You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2023/04/14 19:35:40 UTC
[tika] 01/01: TIKA-4017 -- initial commit
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4017
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ad33f4f83583bc7624bb58690c30a55097ac0ba6
Author: tballison <ta...@apache.org>
AuthorDate: Fri Apr 14 15:35:31 2023 -0400
TIKA-4017 -- initial commit
---
.../main/java/org/apache/tika/metadata/PDF.java | 5 +
.../java/org/apache/tika/parser/pdf/PDFParser.java | 63 ++++++
.../apache/tika/parser/pdf/PDFParserConfig.java | 31 +++
.../tika/parser/pdf/updates/StartXRefOffset.java | 54 +++++
.../tika/parser/pdf/updates/StartXRefScanner.java | 220 +++++++++++++++++++++
.../org/apache/tika/parser/pdf/PDFParserTest.java | 14 ++
.../test-documents/testPDF_IncrementalUpdates.pdf | Bin 0 -> 64872 bytes
7 files changed, 387 insertions(+)
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/PDF.java b/tika-core/src/main/java/org/apache/tika/metadata/PDF.java
index 2b21e2590..add5531b8 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/PDF.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/PDF.java
@@ -27,6 +27,11 @@ public interface PDF {
String PDFA_PREFIX = "pdfa" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
String PDFAID_PREFIX = "pdfaid" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
+
+ Property PDF_INCREMENTAL_UPDATES = Property.externalInteger(PDF_PREFIX + "incrementalUpdates");
+
+ Property EOF_OFFSETS = Property.externalRealSeq(PDF_PREFIX + "eofOffsets");
+
/**
* Prefix to be used for properties that record what was stored
* in the docinfo section (as opposed to XMP)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 785ae615a..fab3eda70 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -19,6 +19,7 @@ package org.apache.tika.parser.pdf;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
@@ -34,6 +35,8 @@ import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.io.RandomAccessBufferedFileInputStream;
+import org.apache.pdfbox.io.RandomAccessRead;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
import org.apache.pdfbox.pdmodel.PDPage;
@@ -69,6 +72,8 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.RenderingParser;
import org.apache.tika.parser.pdf.image.ImageGraphicsEngineFactory;
+import org.apache.tika.parser.pdf.updates.StartXRefOffset;
+import org.apache.tika.parser.pdf.updates.StartXRefScanner;
import org.apache.tika.parser.pdf.xmpschemas.XMPSchemaIllustrator;
import org.apache.tika.renderer.PageRangeRequest;
import org.apache.tika.renderer.RenderResult;
@@ -147,6 +152,7 @@ public class PDFParser extends AbstractParser implements RenderingParser, Initia
PDFRenderingState incomingRenderingState = context.get(PDFRenderingState.class);
TikaInputStream tstream = null;
boolean shouldClose = false;
+ List<StartXRefOffset> xRefOffsets = new ArrayList<>();
try {
if (shouldSpool(localConfig)) {
if (stream instanceof TikaInputStream) {
@@ -159,6 +165,8 @@ public class PDFParser extends AbstractParser implements RenderingParser, Initia
} else {
tstream = TikaInputStream.cast(stream);
}
+ scanXRefOffsets(localConfig, tstream, metadata, xRefOffsets);
+
password = getPassword(metadata, context);
MemoryUsageSetting memoryUsageSetting = MemoryUsageSetting.setupMainMemoryOnly();
if (localConfig.getMaxMainMemoryBytes() >= 0) {
@@ -224,6 +232,42 @@ public class PDFParser extends AbstractParser implements RenderingParser, Initia
}
}
+ private void scanXRefOffsets(PDFParserConfig localConfig,
+ TikaInputStream tikaInputStream,
+ Metadata metadata,
+ List<StartXRefOffset> xRefOffsets) {
+ if (!localConfig.isParseIncrementalUpdates() &&
+ !localConfig.isExtractIncrementalUpdateInfo()) {
+ return;
+ }
+ try (RandomAccessRead ra =
+ new RandomAccessBufferedFileInputStream(tikaInputStream.getFile())) {
+ StartXRefScanner xRefScanner = new StartXRefScanner(ra);
+ xRefOffsets.addAll(xRefScanner.scan());
+ } catch (IOException e) {
+ e.printStackTrace();
+ //swallow
+ }
+
+ int startXrefs = 0;
+ for (StartXRefOffset offset : xRefOffsets) {
+ //don't count linearized dummy xref offset
+ //TODO figure out better way of managing this
+ if (offset.getStartxref() == 0) {
+ continue;
+ }
+ startXrefs++;
+ metadata.add(PDF.EOF_OFFSETS, Long.toString(offset.getEndEofOffset()));
+ }
+
+ if (startXrefs > 0) {
+ //don't count the last xref as an incremental update
+ startXrefs--;
+ }
+ metadata.set(PDF.PDF_INCREMENTAL_UPDATES, startXrefs);
+
+ }
+
private void checkIllustrator(final PDDocument pdfDocument, Metadata metadata) {
PDPage page = null;
@@ -291,6 +335,10 @@ public class PDFParser extends AbstractParser implements RenderingParser, Initia
|| localConfig.getImageStrategy() == PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END) {
return true;
}
+ if (localConfig.isExtractIncrementalUpdateInfo() ||
+ localConfig.isParseIncrementalUpdates()) {
+ return true;
+ }
if (localConfig.getOcrStrategy() == PDFParserConfig.OCR_STRATEGY.NO_OCR) {
return false;
@@ -870,6 +918,21 @@ public class PDFParser extends AbstractParser implements RenderingParser, Initia
defaultConfig.setMaxMainMemoryBytes(maxMainMemoryBytes);
}
+ @Field
+ public void setExtractIncrementalUpdateInfo(boolean setExtractIncrementalUpdateInfo) {
+ defaultConfig.setExtractIncrementalUpdateInfo(setExtractIncrementalUpdateInfo);
+ }
+
+ @Field
+ public void setParseIncrementalUpdateInfo(boolean parseIncrementalUpdateInfo) {
+ defaultConfig.setParseIncrementalUpdates(parseIncrementalUpdateInfo);
+ }
+
+ @Field
+ public void set(int maxIncrementalUpdates) {
+ defaultConfig.setMaxIncrementalUpdates(maxIncrementalUpdates);
+ }
+
public long getMaxMainMemoryBytes() {
return defaultConfig.getMaxMainMemoryBytes();
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index 54776648b..87d2f72f0 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -142,6 +142,12 @@ public class PDFParserConfig implements Serializable {
private Renderer renderer;
+ private boolean extractIncrementalUpdateInfo = true;
+
+ private boolean parseIncrementalUpdates = true;
+
+ int maxIncrementalUpdates = 10;
+
/**
* @return whether or not to extract only inline image metadata and not render the images
*/
@@ -886,7 +892,32 @@ public class PDFParserConfig implements Serializable {
return imageStrategy;
}
+ public boolean isExtractIncrementalUpdateInfo() {
+ return extractIncrementalUpdateInfo;
+ }
+ public void setExtractIncrementalUpdateInfo(boolean extractIncrementalUpdateInfo) {
+ this.extractIncrementalUpdateInfo = extractIncrementalUpdateInfo;
+ userConfigured.add("extractIncrementalUpdateInfo");
+ }
+
+ public boolean isParseIncrementalUpdates() {
+ return parseIncrementalUpdates;
+ }
+
+ public void setParseIncrementalUpdates(boolean parseIncrementalUpdates) {
+ this.parseIncrementalUpdates = parseIncrementalUpdates;
+ userConfigured.add("parseIncrementalUpdates");
+ }
+
+ public int getMaxIncrementalUpdates() {
+ return maxIncrementalUpdates;
+ }
+
+ public void setMaxIncrementalUpdates(int maxIncrementalUpdates) {
+ this.maxIncrementalUpdates = maxIncrementalUpdates;
+ userConfigured.add("maxIncrementalUpdates");
+ }
public enum OCR_STRATEGY {
AUTO, NO_OCR, OCR_ONLY, OCR_AND_TEXT_EXTRACTION;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/updates/StartXRefOffset.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/updates/StartXRefOffset.java
new file mode 100644
index 000000000..ba871b358
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/updates/StartXRefOffset.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf.updates;
+
+public class StartXRefOffset {
+ private final long startxref;
+ private final long startXrefOffset;
+ private final long endEofOffset;
+
+ private final boolean hasEof;
+ public StartXRefOffset(long startxref, long startXrefOffset, long endEofOffset,
+ boolean hasEof) {
+ this.startxref = startxref;
+ this.startXrefOffset = startXrefOffset;
+ this.endEofOffset = endEofOffset;
+ this.hasEof = hasEof;
+ }
+
+ public long getStartxref() {
+ return startxref;
+ }
+
+ public long getStartXrefOffset() {
+ return startXrefOffset;
+ }
+
+ public long getEndEofOffset() {
+ return endEofOffset;
+ }
+
+ public boolean isHasEof() {
+ return hasEof;
+ }
+
+ @Override
+ public String toString() {
+ return "StartXRefOffset{" + "startxref=" + startxref + ", startXrefOffset=" +
+ startXrefOffset + ", endEofOffset=" + endEofOffset + ", hasEof=" + hasEof + '}';
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/updates/StartXRefScanner.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/updates/StartXRefScanner.java
new file mode 100644
index 000000000..b9551e23c
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/updates/StartXRefScanner.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf.updates;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pdfbox.io.RandomAccessRead;
+
+/**
+ * This is a first draft of a scanner to extract incremental updates
+ * out of PDFs. It effectively scans the bytestream looking
+ * for startxref\\s*(\\d+)\\s*(%%EOF\n?)? It does not validate that the
+ * startxrefs point to actual xrefs.
+ */
+public class StartXRefScanner {
+
+ static final int MAX_LENGTH_LONG = Long.toString(Long.MAX_VALUE).length();
+
+ private static final char[] STARTXREF = new char[]{'s', 't', 'a', 'r', 't', 'x', 'r', 'e', 'f'};
+
+ private static final char[] EOF_MARKER = new char[]{'%', '%', 'E', 'O', 'F'};
+
+
+ /**
+ * ASCII code for line feed.
+ */
+ private static final byte ASCII_LF = 10;
+ /**
+ * ASCII code for carriage return.
+ */
+ private static final byte ASCII_CR = 13;
+ private static final byte ASCII_ZERO = 48;
+ private static final byte ASCII_NINE = 57;
+ private static final byte ASCII_SPACE = 32;
+ private final RandomAccessRead source;
+
+ public StartXRefScanner(RandomAccessRead source) {
+ this.source = source;
+ }
+
+ public List<StartXRefOffset> scan() throws IOException {
+ List<StartXRefOffset> offsets = new ArrayList<>();
+ long read = 0;
+ int match = 0;
+ try {
+ int b = source.read();
+ while (b > -1) {
+ if (b == STARTXREF[match]) {
+ match++;
+ if (match >= STARTXREF.length) {
+ long startXREFOffset = source.getPosition() - STARTXREF.length;
+ long startxref = readLong();
+ boolean hasEof = readEOF();
+ long endOfEOFOffset = source.getPosition();
+ offsets.add(new StartXRefOffset(startxref,
+ startXREFOffset, endOfEOFOffset, hasEof));
+ match = 0;
+ }
+ } else {
+ match = 0;
+ }
+ read++;
+ b = source.read();
+ }
+ } finally {
+ if (source.getPosition() >= Integer.MAX_VALUE) {
+ throw new IOException("read more than " + Integer.MAX_VALUE + " bytes");
+ }
+ source.rewind((int)source.getPosition());
+ }
+ return offsets;
+ }
+
+ private boolean readEOF() throws IOException {
+ //this expects %%EOF, with possibly some white space before it
+ //it will fail if there's a comment before %%EOF
+ //TODO -- make this more robust
+ skipWhiteSpaces();
+ int c = source.read();
+ int i = 0;
+ while (c > -1 && c == EOF_MARKER[i] && ++i < EOF_MARKER.length) {
+ c = source.read();
+ }
+
+ if (i == EOF_MARKER.length) {
+ //now look for a single new line following the eof
+ c = source.read();
+ if (c == -1) {
+ //do nothing
+ } else if (isEOL(c)) {
+ //do nothing
+ } else {
+ source.rewind(1);
+ }
+ return true;
+ }
+ if (c == -1) {
+ source.rewind(i - 1);
+ } else {
+ source.rewind(i);
+ }
+ return false;
+ }
+
+ protected void skipWhiteSpaces() throws IOException {
+
+ int whitespace = source.read();
+ while (whitespace > -1 && isWhitespace(whitespace)) {
+
+ whitespace = source.read();
+ }
+ if (whitespace > -1) {
+ source.rewind(1);
+ }
+ }
+
+ protected boolean isWhitespace(int c) {
+ return c == 0 || c == 9 || c == 12 || c == ASCII_LF || c == ASCII_CR || c == ASCII_SPACE;
+ }
+
+ protected long readLong() throws IOException {
+ skipSpaces();
+ long retval = 0;
+
+ StringBuilder longBuffer = readStringNumber();
+
+ try {
+ retval = Long.parseLong(longBuffer.toString());
+ } catch (NumberFormatException e) {
+ source.rewind(longBuffer.toString().getBytes(StandardCharsets.ISO_8859_1).length);
+ throw new IOException("Error: Expected a long type at offset " + source.getPosition() +
+ ", instead got '" + longBuffer + "'", e);
+ }
+ return retval;
+ }
+
+ /**
+ * This will skip all spaces and comments that are present.
+ *
+ * @throws IOException If there is an error reading from the stream.
+ */
+ protected void skipSpaces() throws IOException {
+ int c = source.read();
+ // 37 is the % character, a comment
+ while (isWhitespace(c) || c == 37) {
+ if (c == 37) {
+ // skip past the comment section
+ c = source.read();
+ while (!isEOL(c) && c != -1) {
+ c = source.read();
+ }
+ } else {
+ c = source.read();
+ }
+ }
+ if (c != -1) {
+ source.rewind(1);
+ }
+ }
+
+ /**
+ * This method is used to read a token by the {@linkplain #readLong()} method. Valid
+ * delimiters are any non digit values.
+ *
+ * @return the token to parse as integer or long by the calling method.
+ * @throws IOException throws by the {@link #source} methods.
+ */
+ protected final StringBuilder readStringNumber() throws IOException {
+ int lastByte;
+ StringBuilder buffer = new StringBuilder();
+ while ((lastByte = source.read()) >= '0' && lastByte <= '9') {
+ buffer.append((char) lastByte);
+ if (buffer.length() > MAX_LENGTH_LONG) {
+ throw new IOException(
+ "Number '" + buffer + "' is getting too long, stop reading at offset " +
+ source.getPosition());
+ }
+ }
+ if (lastByte != -1) {
+ source.rewind(1);
+ }
+ return buffer;
+ }
+
+ /**
+ * This will tell if the next byte to be read is an end of line byte.
+ *
+ * @param c The character to check against end of line
+ * @return true if the next byte is 0x0A or 0x0D.
+ */
+ protected boolean isEOL(int c) {
+ return isLF(c) || isCR(c);
+ }
+
+ private boolean isLF(int c) {
+ return ASCII_LF == c;
+ }
+
+ private boolean isCR(int c) {
+ return ASCII_CR == c;
+ }
+}
+
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index ffa05f393..1bdfc82bb 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -1402,6 +1402,20 @@ public class PDFParserTest extends TikaTest {
//components we're looking for.
}
+ @Test
+ public void testIncrementalUpdateInfoExtracted() throws Exception {
+ PDFParserConfig pdfParserConfig = new PDFParserConfig();
+ pdfParserConfig.setExtractIncrementalUpdateInfo(true);
+
+ ParseContext parseContext = new ParseContext();
+ parseContext.set(PDFParserConfig.class, pdfParserConfig);
+ List<Metadata> metadataList = getRecursiveMetadata(
+ "testPDF_IncrementalUpdates.pdf",
+ parseContext);
+ debug(metadataList);
+ assertEquals(2, metadataList.get(0).getInt(PDF.PDF_INCREMENTAL_UPDATES));
+ }
+
/**
* TODO -- need to test signature extraction
*/
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/testPDF_IncrementalUpdates.pdf b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/testPDF_IncrementalUpdates.pdf
new file mode 100644
index 000000000..8494cc839
Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/test-documents/testPDF_IncrementalUpdates.pdf differ