You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/06/03 19:07:43 UTC
[tika] branch main updated: TIKA-3437 -- remove the preflight
parser from 2.x
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 0a84892 TIKA-3437 -- remove the preflight parser from 2.x
0a84892 is described below
commit 0a84892e33bb8ba69dcfd7d4928831c85af102a0
Author: tballison <ta...@apache.org>
AuthorDate: Thu Jun 3 15:07:31 2021 -0400
TIKA-3437 -- remove the preflight parser from 2.x
---
CHANGES.txt | 5 +
tika-bundles/tika-bundle-standard/pom.xml | 2 -
.../main/java/org/apache/tika/metadata/PDF.java | 17 --
.../tika-parser-pdf-module/pom.xml | 5 -
.../apache/tika/parser/pdf/PDFPreflightParser.java | 180 ---------------------
.../tika/parser/pdf/PDFPreflightParserTest.java | 60 -------
.../tika/parser/pdf/tika-preflight-config.xml | 25 ---
7 files changed, 5 insertions(+), 289 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f8ae159..db90112 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+Release 2.0.0 - ???
+
+ * Cleanup of fetcher integration with tika-server.
+
Release 2.0.0-BETA - 05/19/2021
* Refactor pipes module for resilience
@@ -14,6 +18,7 @@ Release 2.0.0-ALPHA - 01/13/2021
* By default, when rendering a page for OCR, the PDFParser does not render glyphs/text.
* tika-server's "enableFileUrl" has been removed in favor of a FileSystemFetcher.
* Removed deprecated Metadata keys/properties (TIKA-1974).
+ * Removed deprecated PDFPreflightParser (TIKA-3437).
* Removed dangerous calls to read an inputstream or convert to bytes
without specifying a charset
* Parsers can be configured via tika-config.xml on instantiation.
diff --git a/tika-bundles/tika-bundle-standard/pom.xml b/tika-bundles/tika-bundle-standard/pom.xml
index fce99c8..9f9f003 100644
--- a/tika-bundles/tika-bundle-standard/pom.xml
+++ b/tika-bundles/tika-bundle-standard/pom.xml
@@ -198,7 +198,6 @@
pdfbox|
pdfbox-tools|
pdfbox-debugger|
- pdfbox-preflight|
fontbox|
jempbox|
bcmail-jdk15on|
@@ -316,7 +315,6 @@
org.apache.commons.exec;resolution:=optional,
org.apache.jcp.xml.dsig.internal.dom;resolution:=optional,
org.apache.pdfbox.debugger;resolution:=optional,
- org.apache.pdfbox.preflight.*;resolution:=optional,
org.apache.tools.ant;resolution:=optional,
org.apache.tools.ant.taskdefs;resolution:=optional,
org.apache.tools.ant.types;resolution:=optional,
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/PDF.java b/tika-core/src/main/java/org/apache/tika/metadata/PDF.java
index 9f0fdc4..887be5b 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/PDF.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/PDF.java
@@ -26,7 +26,6 @@ public interface PDF {
String PDF_PREFIX = "pdf" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
String PDFA_PREFIX = "pdfa" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
String PDFAID_PREFIX = "pdfaid" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
- String PDF_PREFLIGHT_PREFIX = "pdf-preflight" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
/**
* Prefix to be used for properties that record what was stored
@@ -102,20 +101,4 @@ public interface PDF {
Property HAS_MARKED_CONTENT = Property.internalBoolean(PDF_PREFIX + "hasMarkedContent");
- Property PREFLIGHT_IS_VALID = Property.internalBoolean(PDF_PREFLIGHT_PREFIX + "isValid");
- Property PREFLIGHT_PARSE_EXCEPTION =
- Property.internalText(PDF_PREFLIGHT_PREFIX + "parseException");
- Property PREFLIGHT_VALIDATION_ERRORS =
- Property.internalTextBag(PDF_PREFLIGHT_PREFIX + "validationErrors");
- Property PREFLIGHT_SPECIFICATION =
- Property.internalText(PDF_PREFLIGHT_PREFIX + "specification");
- Property PREFLIGHT_TRAILER_COUNT =
- Property.internalInteger(PDF_PREFLIGHT_PREFIX + "trailerCount");
- Property PREFLIGHT_XREF_TYPE = Property.internalText(PDF_PREFLIGHT_PREFIX + "xrefType");
- Property PREFLIGHT_ICC_PROFILE = Property.internalText(PDF_PREFLIGHT_PREFIX + "iccProfile");
- Property PREFLIGHT_IS_LINEARIZED =
- Property.internalBoolean(PDF_PREFLIGHT_PREFIX + "isLinearized");
-
- Property PREFLIGHT_INCREMENTAL_UPDATES =
- Property.internalBoolean(PDF_PREFLIGHT_PREFIX + "hasIncrementalUpdates");
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
index 59b987c..05e66b0 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
@@ -47,11 +47,6 @@
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
- <artifactId>preflight</artifactId>
- <version>${pdfbox.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.pdfbox</groupId>
<artifactId>jempbox</artifactId>
<version>${jempbox.version}</version>
</dependency>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFPreflightParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFPreflightParser.java
deleted file mode 100644
index 0bbca46..0000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFPreflightParser.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tika.parser.pdf;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Path;
-import java.util.List;
-
-import org.apache.pdfbox.cos.COSBase;
-import org.apache.pdfbox.cos.COSDictionary;
-import org.apache.pdfbox.cos.COSDocument;
-import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.cos.COSObject;
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdfparser.XrefTrailerResolver;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.preflight.Format;
-import org.apache.pdfbox.preflight.PreflightConfiguration;
-import org.apache.pdfbox.preflight.PreflightConstants;
-import org.apache.pdfbox.preflight.PreflightContext;
-import org.apache.pdfbox.preflight.PreflightDocument;
-import org.apache.pdfbox.preflight.ValidationResult;
-import org.apache.pdfbox.preflight.exception.SyntaxValidationException;
-import org.apache.pdfbox.preflight.parser.PreflightParser;
-
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.PDF;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.utils.ExceptionUtils;
-
-public class PDFPreflightParser extends PDFParser {
-
- private static final PDFPreflightParserConfig DEFAULT = new PDFPreflightParserConfig();
-
- /**
- * Copied verbatim from PDFBox
- * <p>
- * According to the PDF Reference, A linearized PDF contain a dictionary as first object
- * (linearized dictionary) and
- * only this one in the first section.
- *
- * @param document the document to validate.
- * @return the linearization dictionary or null.
- */
- protected static COSDictionary getLinearizedDictionary(PDDocument document) {
- // ---- Get Ref to obj
- COSDocument cDoc = document.getDocument();
- List<?> lObj = cDoc.getObjects();
- for (Object object : lObj) {
- COSBase curObj = ((COSObject) object).getObject();
- if (curObj instanceof COSDictionary && ((COSDictionary) curObj).keySet()
- .contains(COSName.getPDFName(PreflightConstants.DICTIONARY_KEY_LINEARIZED))) {
- return (COSDictionary) curObj;
- }
- }
- return null;
- }
-
- @Override
- protected PDDocument getPDDocument(InputStream inputStream, String password,
- MemoryUsageSetting memoryUsageSetting, Metadata metadata,
- ParseContext parseContext) throws IOException {
- try (TikaInputStream tis = TikaInputStream.get(inputStream)) {
- return getPDDocument(tis.getPath(), password, memoryUsageSetting, metadata,
- parseContext);
- }
- }
-
- @Override
- protected PDDocument getPDDocument(Path path, String password,
- MemoryUsageSetting memoryUsageSetting, Metadata metadata,
- ParseContext context) throws IOException {
- PDFPreflightParserConfig pppConfig = context.get(PDFPreflightParserConfig.class, DEFAULT);
-
- PreflightConfiguration configuration = new PreflightConfiguration();
- configuration.setMaxErrors(pppConfig.getMaxErrors());
- PreflightParser preflightParser = new PreflightParser(path.toFile());
-
- preflightParser.setLenient(pppConfig.isLenient);
- try {
- preflightParser.parse(pppConfig.getFormat(), configuration);
- } catch (SyntaxValidationException e) {
- //back off to try to load the file normally
- return handleSyntaxException(path, password, memoryUsageSetting, metadata, e);
- }
-
- PreflightDocument preflightDocument = preflightParser.getPreflightDocument();
- preflightDocument.validate();
- extractPreflight(preflightDocument, metadata);
-
- //need to return this to ensure that it gets closed
- //the preflight document can keep some other resources open.
- return preflightParser.getPreflightDocument();
- }
-
- private void extractPreflight(PreflightDocument preflightDocument, Metadata metadata) {
- ValidationResult result = preflightDocument.getResult();
- metadata.set(PDF.PREFLIGHT_SPECIFICATION, preflightDocument.getSpecification().toString());
- metadata.set(PDF.PREFLIGHT_IS_VALID, Boolean.toString(result.isValid()));
-
-
- List<ValidationResult.ValidationError> errors = result.getErrorsList();
- for (ValidationResult.ValidationError err : errors) {
- metadata.add(PDF.PREFLIGHT_VALIDATION_ERRORS,
- err.getErrorCode() + " : " + err.getDetails());
- }
-
- PreflightContext preflightContext = preflightDocument.getContext();
-
- XrefTrailerResolver resolver = preflightContext.getXrefTrailerResolver();
- int trailerCount = resolver.getTrailerCount();
-
- metadata.set(PDF.PREFLIGHT_TRAILER_COUNT, trailerCount);
- metadata.set(PDF.PREFLIGHT_XREF_TYPE, resolver.getXrefType().toString());
- if (preflightContext.getIccProfileWrapper() != null &&
- preflightContext.getIccProfileWrapper().getProfile() != null) {
- metadata.set(PDF.PREFLIGHT_ICC_PROFILE,
- preflightContext.getIccProfileWrapper().getProfile().toString());
- }
- COSDictionary linearized = getLinearizedDictionary(preflightDocument);
- if (linearized != null) {
- metadata.set(PDF.PREFLIGHT_IS_LINEARIZED, "true");
- if (trailerCount > 2) {
- metadata.set(PDF.PREFLIGHT_INCREMENTAL_UPDATES, "true");
- } else {
- metadata.set(PDF.PREFLIGHT_INCREMENTAL_UPDATES, "false");
- }
- } else {
- metadata.set(PDF.PREFLIGHT_IS_LINEARIZED, "false");
- if (trailerCount > 1) {
- metadata.set(PDF.PREFLIGHT_INCREMENTAL_UPDATES, "true");
- } else {
- metadata.set(PDF.PREFLIGHT_INCREMENTAL_UPDATES, "false");
- }
- }
- }
-
- private PDDocument handleSyntaxException(Path path, String password,
- MemoryUsageSetting memoryUsageSetting,
- Metadata metadata, SyntaxValidationException e)
- throws IOException {
- metadata.add(PDF.PREFLIGHT_PARSE_EXCEPTION, ExceptionUtils.getStackTrace(e));
- return PDDocument.load(path.toFile(), password, memoryUsageSetting);
- }
-
- private static class PDFPreflightParserConfig {
- private int maxErrors = 100;
- private boolean isLenient = true;
- private Format format = Format.PDF_A1B;
-
- public int getMaxErrors() {
- return maxErrors;
- }
-
- public boolean isLenient() {
- return isLenient;
- }
-
- public Format getFormat() {
- return format;
- }
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPreflightParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPreflightParserTest.java
deleted file mode 100644
index cf8d338..0000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPreflightParserTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.pdf;
-
-import static junit.framework.TestCase.assertEquals;
-
-import java.io.InputStream;
-import java.util.List;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.PDF;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.Parser;
-
-public class PDFPreflightParserTest extends TikaTest {
-
- private static Parser PREFLIGHT_AUTO_DETECT_PARSER;
-
- @BeforeClass
- public static void setUp() throws Exception {
- try (InputStream is = PDFPreflightParser.class
- .getResourceAsStream("tika-preflight-config.xml")) {
- PREFLIGHT_AUTO_DETECT_PARSER = new AutoDetectParser(new TikaConfig(is).getParser());
- }
- }
-
- @Test
- public void testBasic() throws Exception {
- List<Metadata> metadataList = getRecursiveMetadata("testPDFFileEmbInAnnotation.pdf",
- PREFLIGHT_AUTO_DETECT_PARSER);
- assertEquals(2, metadataList.size());
-
- Metadata m = metadataList.get(0);
- assertEquals("true", m.get(PDF.PREFLIGHT_IS_LINEARIZED));
- assertEquals("true", m.get(PDF.PREFLIGHT_IS_VALID));
- assertEquals("PDF_A1B", m.get(PDF.PREFLIGHT_SPECIFICATION));
- assertEquals("2", m.get(PDF.PREFLIGHT_TRAILER_COUNT));
- assertEquals("STREAM", m.get(PDF.PREFLIGHT_XREF_TYPE));
- assertEquals("false", m.get(PDF.PREFLIGHT_INCREMENTAL_UPDATES));
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/org/apache/tika/parser/pdf/tika-preflight-config.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/org/apache/tika/parser/pdf/tika-preflight-config.xml
deleted file mode 100644
index 67c02f4..0000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/resources/org/apache/tika/parser/pdf/tika-preflight-config.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<properties>
- <parsers>
- <parser class="org.apache.tika.parser.DefaultParser">
- <parser-exclude class="org.apache.tika.parser.pdf.PDFParser"/>
- </parser>
- <parser class="org.apache.tika.parser.pdf.PDFPreflightParser"/>
- </parsers>
-</properties>