You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2023/03/31 20:12:43 UTC
[tika] 01/01: TIKA-4007 -- allow users to turn off digesting of the container document
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4007
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6cdc65a592b3380034f98194fa86474897d9bc00
Author: tballison <ta...@apache.org>
AuthorDate: Fri Mar 31 16:12:30 2023 -0400
TIKA-4007 -- allow users to turn off digesting of the container document
---
.../org/apache/tika/parser/AutoDetectParser.java | 6 ++--
.../org/apache/tika/parser/DigestingParser.java | 24 ++++++++++++++--
.../parser/digestutils/CommonsDigesterFactory.java | 13 +++++++++
.../tika/parser/AutoDetectParserConfigTest.java | 21 ++++++++++++++
.../parser/BouncyCastleDigestingParserTest.java | 10 +++----
.../apache/tika/parser/DigestingParserTest.java | 11 +++++---
.../tika/parser/RecursiveParserWrapperTest.java | 2 +-
.../configs/tika-config-digests-skip-container.xml | 33 ++++++++++++++++++++++
8 files changed, 105 insertions(+), 15 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
index 491ad572e..45e972c20 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
@@ -109,7 +109,8 @@ public class AutoDetectParser extends CompositeParser {
return fallback;
} else {
return new DigestingParser(fallback,
- config.getAutoDetectParserConfig().getDigesterFactory().build());
+ config.getAutoDetectParserConfig().getDigesterFactory().build(),
+ config.getAutoDetectParserConfig().getDigesterFactory().isSkipContainerDocument());
}
}
@@ -119,7 +120,8 @@ public class AutoDetectParser extends CompositeParser {
return config.getParser();
}
return new DigestingParser(config.getParser(),
- config.getAutoDetectParserConfig().getDigesterFactory().build());
+ config.getAutoDetectParserConfig().getDigesterFactory().build(),
+ config.getAutoDetectParserConfig().getDigesterFactory().isSkipContainerDocument());
}
/**
diff --git a/tika-core/src/main/java/org/apache/tika/parser/DigestingParser.java b/tika-core/src/main/java/org/apache/tika/parser/DigestingParser.java
index 30736091a..24eee2017 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/DigestingParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/DigestingParser.java
@@ -21,6 +21,7 @@ package org.apache.tika.parser;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -32,15 +33,16 @@ import org.apache.tika.metadata.Metadata;
public class DigestingParser extends ParserDecorator {
private final Digester digester;
-
+ private final boolean skipContainerDocument;
/**
* Creates a decorator for the given parser.
*
* @param parser the parser instance to be decorated
*/
- public DigestingParser(Parser parser, Digester digester) {
+ public DigestingParser(Parser parser, Digester digester, boolean skipContainerDocument) {
super(parser);
this.digester = digester;
+ this.skipContainerDocument = skipContainerDocument;
}
@Override
@@ -49,7 +51,7 @@ public class DigestingParser extends ParserDecorator {
TemporaryResources tmp = new TemporaryResources();
TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata);
try {
- if (digester != null) {
+ if (shouldDigest(metadata)) {
digester.digest(tis, metadata, context);
}
super.parse(tis, handler, metadata, context);
@@ -58,12 +60,28 @@ public class DigestingParser extends ParserDecorator {
}
}
+ private boolean shouldDigest(Metadata metadata) {
+ if (digester == null) {
+ return false;
+ }
+ if (! skipContainerDocument) {
+ return true;
+ }
+ Integer parseDepth = metadata.getInt(TikaCoreProperties.EMBEDDED_DEPTH);
+ if (parseDepth == null || parseDepth == 0) {
+ return false;
+ }
+ return true;
+ }
+
/**
* This is used in {@link AutoDetectParserConfig} to (optionally)
* wrap the parser in a digesting parser.
*/
public interface DigesterFactory {
Digester build();
+ void setSkipContainerDocument(boolean skipContainerDocument);
+ boolean isSkipContainerDocument();
}
/**
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-digest-commons/src/main/java/org/apache/tika/parser/digestutils/CommonsDigesterFactory.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-digest-commons/src/main/java/org/apache/tika/parser/digestutils/CommonsDigesterFactory.java
index 17d0765bc..d37f7acb1 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-digest-commons/src/main/java/org/apache/tika/parser/digestutils/CommonsDigesterFactory.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-digest-commons/src/main/java/org/apache/tika/parser/digestutils/CommonsDigesterFactory.java
@@ -28,6 +28,8 @@ public class CommonsDigesterFactory implements DigestingParser.DigesterFactory {
private int markLimit = 1000000;
private String algorithmString = "md5";
+ private boolean skipContainerDocument = false;
+
@Override
public DigestingParser.Digester build() {
return new CommonsDigester(markLimit, algorithmString);
@@ -42,4 +44,15 @@ public class CommonsDigesterFactory implements DigestingParser.DigesterFactory {
public void setAlgorithmString(String algorithmString) {
this.algorithmString = algorithmString;
}
+
+ @Field
+ @Override
+ public void setSkipContainerDocument(boolean skipContainerDocument) {
+ this.skipContainerDocument = skipContainerDocument;
+ }
+
+ @Override
+ public boolean isSkipContainerDocument() {
+ return skipContainerDocument;
+ }
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserConfigTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserConfigTest.java
index 7ef747157..f1efedb59 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserConfigTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/AutoDetectParserConfigTest.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.InputStream;
import java.nio.file.Files;
@@ -124,6 +125,26 @@ public class AutoDetectParserConfigTest extends TikaTest {
metadataList.get(6).get("X-TIKA:digest:MD5"));
}
+ @Test
+ public void testDigestsSkipContainer() throws Exception {
+ //test to make sure that the decorator is only applied once for
+ //legacy (e.g. not RecursiveParserWrapperHandler) parsing
+ TikaConfig tikaConfig = null;
+ try (InputStream is = AutoDetectParserConfigTest.class.getResourceAsStream(
+ "/configs/tika-config-digests-skip-container.xml")) {
+ tikaConfig = new TikaConfig(is);
+ }
+ Parser p = new AutoDetectParser(tikaConfig);
+ List<Metadata> metadataList = getRecursiveMetadata("testPPT_EmbeddedPDF.pptx", p);
+ assertNull(metadataList.get(0).get("X-TIKA:digest:SHA256"));
+ assertNull(metadataList.get(0).get("X-TIKA:digest:MD5"));
+
+ assertEquals("Q7D3RFV6DNGZ4BQIS6UKNWX4CDIKPIGDU2D7ADBUDVOBYSZHF7FQ====",
+ metadataList.get(6).get("X-TIKA:digest:SHA256"));
+ assertEquals("90a8b249a6d6b6cb127c59e01cef3aaa",
+ metadataList.get(6).get("X-TIKA:digest:MD5"));
+ }
+
@Test
public void testDigestsEmptyParser() throws Exception {
//TIKA-3939 -- ensure that digesting happens even with EmptyParser
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/BouncyCastleDigestingParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/BouncyCastleDigestingParserTest.java
index 474ba7717..9971b7e03 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/BouncyCastleDigestingParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/BouncyCastleDigestingParserTest.java
@@ -76,7 +76,7 @@ public class BouncyCastleDigestingParserTest extends TikaTest {
Metadata m = new Metadata();
XMLResult xml = getXML("test_recursive_embedded.docx",
new DigestingParser(AUTO_DETECT_PARSER,
- new BouncyCastleDigester(UNLIMITED, algo)), m);
+ new BouncyCastleDigester(UNLIMITED, algo), false), m);
assertEquals(expected.get(algo), m.get(P + algo));
}
@@ -105,7 +105,7 @@ public class BouncyCastleDigestingParserTest extends TikaTest {
Metadata m = new Metadata();
XMLResult xml = getXML("test_recursive_embedded.docx",
new DigestingParser(AUTO_DETECT_PARSER, new BouncyCastleDigester(UNLIMITED,
- "MD5,SHA256,SHA384,SHA512,SHA3-512,SHA1:32")), m);
+ "MD5,SHA256,SHA384,SHA512,SHA3-512,SHA1:32"), false), m);
for (String algo : new String[]{"MD5", "SHA256", "SHA384", "SHA512", "SHA3-512", "SHA1"}) {
assertEquals(expected.get(algo), m.get(P + algo));
}
@@ -119,7 +119,7 @@ public class BouncyCastleDigestingParserTest extends TikaTest {
String expectedMD5 = "59f626e09a8c16ab6dbc2800c685f772";
Metadata m = new Metadata();
XMLResult xml = getXML("test_recursive_embedded.docx",
- new DigestingParser(AUTO_DETECT_PARSER, new BouncyCastleDigester(100, "MD5")), m);
+ new DigestingParser(AUTO_DETECT_PARSER, new BouncyCastleDigester(100, "MD5"), false), m);
assertEquals(expectedMD5, m.get(P + "MD5"));
}
@@ -127,7 +127,7 @@ public class BouncyCastleDigestingParserTest extends TikaTest {
public void testNegativeMaxMarkLength() throws Exception {
assertThrows(IllegalArgumentException.class, () -> {
getXML("test_recursive_embedded.docx",
- new DigestingParser(AUTO_DETECT_PARSER, new BouncyCastleDigester(-1, "MD5")));
+ new DigestingParser(AUTO_DETECT_PARSER, new BouncyCastleDigester(-1, "MD5"), false));
});
}
@@ -135,7 +135,7 @@ public class BouncyCastleDigestingParserTest extends TikaTest {
public void testUnrecognizedEncodingOptions() throws Exception {
assertThrows(IllegalArgumentException.class, () -> {
getXML("test_recursive_embedded.docx", new DigestingParser(AUTO_DETECT_PARSER,
- new BouncyCastleDigester(100000, "MD5:33")));
+ new BouncyCastleDigester(100000, "MD5:33"), false));
});
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/DigestingParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/DigestingParserTest.java
index 2a377c7f5..508d6f4d7 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/DigestingParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/DigestingParserTest.java
@@ -75,7 +75,7 @@ public class DigestingParserTest extends TikaTest {
for (CommonsDigester.DigestAlgorithm algo : CommonsDigester.DigestAlgorithm.values()) {
Metadata m = new Metadata();
XMLResult xml = getXML("test_recursive_embedded.docx",
- new DigestingParser(AUTO_DETECT_PARSER, new CommonsDigester(UNLIMITED, algo)),
+ new DigestingParser(AUTO_DETECT_PARSER, new CommonsDigester(UNLIMITED, algo), false),
m);
assertEquals(expected.get(algo), m.get(P + algo.toString()), algo.toString());
}
@@ -103,7 +103,8 @@ public class DigestingParserTest extends TikaTest {
Metadata m = new Metadata();
XMLResult xml = getXML("test_recursive_embedded.docx",
new DigestingParser(AUTO_DETECT_PARSER,
- new CommonsDigester(UNLIMITED, "md5,sha256,sha384,sha512,sha1:32")), m);
+ new CommonsDigester(UNLIMITED, "md5,sha256,sha384,sha512,sha1:32"), false)
+ , m);
for (CommonsDigester.DigestAlgorithm algo : new CommonsDigester.DigestAlgorithm[]{
CommonsDigester.DigestAlgorithm.MD5, CommonsDigester.DigestAlgorithm.SHA1,
CommonsDigester.DigestAlgorithm.SHA256, CommonsDigester.DigestAlgorithm.SHA384,
@@ -120,7 +121,8 @@ public class DigestingParserTest extends TikaTest {
Metadata m = new Metadata();
XMLResult xml = getXML("test_recursive_embedded.docx",
new DigestingParser(AUTO_DETECT_PARSER,
- new CommonsDigester(100, CommonsDigester.DigestAlgorithm.MD5)), m);
+ new CommonsDigester(100, CommonsDigester.DigestAlgorithm.MD5),false)
+ , m);
assertEquals(expectedMD5, m.get(P + "MD5"));
}
@@ -131,7 +133,8 @@ public class DigestingParserTest extends TikaTest {
try {
XMLResult xml = getXML("test_recursive_embedded.docx",
new DigestingParser(AUTO_DETECT_PARSER,
- new CommonsDigester(-1, CommonsDigester.DigestAlgorithm.MD5)), m);
+ new CommonsDigester(-1, CommonsDigester.DigestAlgorithm.MD5),
+ false), m);
} catch (IllegalArgumentException e) {
ex = true;
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java
index 28b22a29a..24800926a 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java
@@ -392,7 +392,7 @@ public class RecursiveParserWrapperTest extends TikaTest {
ParseContext context = new ParseContext();
Parser wrapped = AUTO_DETECT_PARSER;
if (digester != null) {
- wrapped = new DigestingParser(wrapped, digester);
+ wrapped = new DigestingParser(wrapped, digester, false);
}
RecursiveParserWrapper wrapper =
new RecursiveParserWrapper(wrapped, catchEmbeddedExceptions);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-digests-skip-container.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-digests-skip-container.xml
new file mode 100644
index 000000000..22823dc3c
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/resources/configs/tika-config-digests-skip-container.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <parsers>
+ <parser class="org.apache.tika.parser.DefaultParser"/>
+ </parsers>
+ <autoDetectParserConfig>
+ <spoolToDisk>1000000</spoolToDisk>
+ <outputThreshold>1000000</outputThreshold>
+ <digesterFactory
+ class="org.apache.tika.parser.digestutils.CommonsDigesterFactory">
+ <markLimit>100000</markLimit>
+ <algorithmString>sha256:32,md5</algorithmString>
+ <skipContainerDocument>true</skipContainerDocument>
+ </digesterFactory>
+ <throwOnZeroBytes>false</throwOnZeroBytes>
+ </autoDetectParserConfig>
+</properties>