You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2020/03/06 21:07:53 UTC
[tika] branch branch_1x updated (5276459 -> e38f609)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 5276459 TIKA-3042 -- allow custom overriding of date formats in .xls and .xlsx
new f9e64bc TIKA-3059 -- prevent NPE in new inline image processing code copied from PDFBox
new 8a2cf1e TIKA-3061 -- need to read inputstream completely before processing in StreamingZipContainerDetector
new 26c320d TIKA-3062 -- improve alignment of attachments based on digests
new e38f609 clean accidental addition
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/test/java/org/apache/tika/TikaTest.java | 5 +
.../org/apache/tika/eval/AbstractProfiler.java | 8 +-
.../java/org/apache/tika/eval/ExtractComparer.java | 121 ++++++++++++++-------
.../eval/textstats/ContentLengthCalculator.java | 2 +-
.../org/apache/tika/eval/SimpleComparerTest.java | 3 +-
.../extractsA/file14_diffAttachOrder.json | 27 ++---
.../extractsB/file14_diffAttachOrder.json | 26 +++--
.../tika/parser/pdf/ImageGraphicsEngine.java | 30 ++---
.../parser/pkg/StreamingZipContainerDetector.java | 10 +-
.../tika/detect/TestContainerAwareDetector.java | 9 ++
.../test-documents/testOpenOfficeInAZip.zip | Bin 0 -> 10706 bytes
11 files changed, 156 insertions(+), 85 deletions(-)
create mode 100644 tika-parsers/src/test/resources/test-documents/testOpenOfficeInAZip.zip
[tika] 01/04: TIKA-3059 -- prevent NPE in new inline image
processing code copied from PDFBox
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit f9e64bc9ef95aa1d2f9e9b334fdccfefc94c523b
Author: tallison <ta...@apache.org>
AuthorDate: Fri Mar 6 10:36:08 2020 -0500
TIKA-3059 -- prevent NPE in new inline image processing code copied from PDFBox
---
.../tika/parser/pdf/ImageGraphicsEngine.java | 30 ++++++++++++----------
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/ImageGraphicsEngine.java b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/ImageGraphicsEngine.java
index f47ae73..3f0c9d0 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pdf/ImageGraphicsEngine.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pdf/ImageGraphicsEngine.java
@@ -33,6 +33,7 @@ import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
import org.apache.pdfbox.pdmodel.graphics.pattern.PDAbstractPattern;
import org.apache.pdfbox.pdmodel.graphics.pattern.PDTilingPattern;
+import org.apache.pdfbox.pdmodel.graphics.state.PDExtendedGraphicsState;
import org.apache.pdfbox.pdmodel.graphics.state.PDSoftMask;
import org.apache.pdfbox.pdmodel.graphics.state.RenderingMode;
import org.apache.pdfbox.tools.imageio.ImageIOUtil;
@@ -118,20 +119,23 @@ class ImageGraphicsEngine extends PDFGraphicsStreamEngine {
}
for (COSName name : res.getExtGStateNames()) {
- PDSoftMask softMask = res.getExtGState(name).getSoftMask();
-
- if (softMask != null) {
- try {
- PDTransparencyGroup group = softMask.getGroup();
-
- if (group != null) {
- // PDFBOX-4327: without this line NPEs will occur
- res.getExtGState(name).copyIntoGraphicsState(getGraphicsState());
-
- processSoftMask(group);
+ PDExtendedGraphicsState extendedGraphicsState = res.getExtGState(name);
+ if (extendedGraphicsState != null) {
+ PDSoftMask softMask = extendedGraphicsState.getSoftMask();
+
+ if (softMask != null) {
+ try {
+ PDTransparencyGroup group = softMask.getGroup();
+
+ if (group != null) {
+ // PDFBOX-4327: without this line NPEs will occur
+ res.getExtGState(name).copyIntoGraphicsState(getGraphicsState());
+
+ processSoftMask(group);
+ }
+ } catch (IOException e) {
+ handleCatchableIOE(e);
}
- } catch (IOException e) {
- handleCatchableIOE(e);
}
}
}
[tika] 04/04: clean accidental addition
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit e38f60903fa4f8b4dc79c99660a61792a3e256b8
Author: tallison <ta...@apache.org>
AuthorDate: Fri Mar 6 14:51:28 2020 -0500
clean accidental addition
---
bin/jhove-apps-1.22.0.jar | Bin 321216 -> 0 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/bin/jhove-apps-1.22.0.jar b/bin/jhove-apps-1.22.0.jar
deleted file mode 100644
index a90b104..0000000
Binary files a/bin/jhove-apps-1.22.0.jar and /dev/null differ
[tika] 03/04: TIKA-3062 -- improve alignment of attachments based
on digests
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 26c320d470c5449a67d5a06d3375e1a4d6ba30ed
Author: tallison <ta...@apache.org>
AuthorDate: Fri Mar 6 14:41:33 2020 -0500
TIKA-3062 -- improve alignment of attachments based on digests
---
bin/jhove-apps-1.22.0.jar | Bin 0 -> 321216 bytes
.../org/apache/tika/eval/AbstractProfiler.java | 8 +-
.../java/org/apache/tika/eval/ExtractComparer.java | 121 ++++++++++++++-------
.../eval/textstats/ContentLengthCalculator.java | 2 +-
.../org/apache/tika/eval/SimpleComparerTest.java | 3 +-
.../extractsA/file14_diffAttachOrder.json | 27 ++---
.../extractsB/file14_diffAttachOrder.json | 26 +++--
7 files changed, 116 insertions(+), 71 deletions(-)
diff --git a/bin/jhove-apps-1.22.0.jar b/bin/jhove-apps-1.22.0.jar
new file mode 100644
index 0000000..a90b104
Binary files /dev/null and b/bin/jhove-apps-1.22.0.jar differ
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
index e23361b..b54b242 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
@@ -390,9 +390,11 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
Map<Cols, String> data = new HashMap<>();
data.put(Cols.ID, fileId);
if (textStats.containsKey(ContentLengthCalculator.class)) {
- data.put(Cols.CONTENT_LENGTH, Integer.toString((Integer) textStats.get(ContentLengthCalculator.class)));
- } else {
- data.put(Cols.CONTENT_LENGTH, "0");
+ int length = (int)textStats.get(ContentLengthCalculator.class);
+ if (length == 0) {
+ return;
+ }
+ data.put(Cols.CONTENT_LENGTH, Integer.toString(length));
}
langid(textStats, data);
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
index f314f53..91568db 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/ExtractComparer.java
@@ -277,6 +277,8 @@ public class ExtractComparer extends AbstractProfiler {
}
List<Integer> numAttachmentsA = countAttachments(metadataListA);
List<Integer> numAttachmentsB = countAttachments(metadataListB);
+
+ String sharedDigestKey = findSharedDigestKey(metadataListA, metadataListB);
Map<Class, Object> tokenStatsA = null;
Map<Class, Object> tokenStatsB = null;
//now get that metadata
@@ -294,7 +296,8 @@ public class ExtractComparer extends AbstractProfiler {
writeProfileData(fpsA, i, contentTagsA, metadataA, fileId, containerID, numAttachmentsA, PROFILES_A);
writeExceptionData(fileId, metadataA, EXCEPTION_TABLE_A);
- int matchIndex = getMatch(i, metadataListA, metadataListB);
+ int matchIndex = getMatch(i, sharedDigestKey,
+ handledB, metadataListA, metadataListB);
if (matchIndex > -1 && ! handledB.contains(matchIndex)) {
metadataB = metadataListB.get(matchIndex);
@@ -310,28 +313,33 @@ public class ExtractComparer extends AbstractProfiler {
//write content
try {
tokenStatsA = calcTextStats(contentTagsA);
- tokenStatsB = calcTextStats(contentTagsB);
writeContentData(fileId, tokenStatsA, CONTENTS_TABLE_A);
- writeContentData(fileId, tokenStatsB, CONTENTS_TABLE_B);
+ tokenStatsB = calcTextStats(contentTagsB);
+ if (metadataB != null) {
+ writeContentData(fileId, tokenStatsB, CONTENTS_TABLE_B);
+ }
} catch (IOException e) {
throw new RuntimeException(e);
}
-
- TokenCounts tokenCountsA = (TokenCounts)tokenStatsA.get(BasicTokenCountStatsCalculator.class);
- TokenCounts tokenCountsB = (TokenCounts)tokenStatsB.get(BasicTokenCountStatsCalculator.class);
- //now run comparisons
- if (tokenCountsA.getTotalTokens() > 0
- && tokenCountsB.getTotalTokens() > 0) {
- Map<Cols, String> data = new HashMap<>();
- data.put(Cols.ID, fileId);
-
- ContrastStatistics contrastStatistics =
- tokenContraster.calculateContrastStatistics(
- tokenCountsA,
- tokenCountsB);
-
- writeContrasts(data, contrastStatistics);
- writer.writeRow(CONTENT_COMPARISONS, data);
+ if (metadataB != null) {
+ TokenCounts tokenCountsA = (TokenCounts) tokenStatsA.get(BasicTokenCountStatsCalculator.class);
+ TokenCounts tokenCountsB = (TokenCounts) tokenStatsB.get(BasicTokenCountStatsCalculator.class);
+ //arbitrary decision...only run the comparisons if there are > 10 tokens total
+ //We may want to bump that value a bit higher?
+ //now run comparisons
+ if (tokenCountsA.getTotalTokens()
+ + tokenCountsB.getTotalTokens() > 10) {
+ Map<Cols, String> data = new HashMap<>();
+ data.put(Cols.ID, fileId);
+
+ ContrastStatistics contrastStatistics =
+ tokenContraster.calculateContrastStatistics(
+ tokenCountsA,
+ tokenCountsB);
+
+ writeContrasts(data, contrastStatistics);
+ writer.writeRow(CONTENT_COMPARISONS, data);
+ }
}
}
}
@@ -362,6 +370,34 @@ public class ExtractComparer extends AbstractProfiler {
}
}
+ /**
+ * Checks only the first item in each list. Returns the first
+ * digest key shared by both, if it exists, null otherwise.
+ * @param metadataListA
+ * @param metadataListB
+ * @return
+ */
+ private String findSharedDigestKey(List<Metadata> metadataListA, List<Metadata> metadataListB) {
+ if (metadataListB == null || metadataListB.size() == 0) {
+ return null;
+ }
+ Set<String> digestA = new HashSet<>();
+ if (metadataListA != null) {
+ for (String n : metadataListA.get(0).names()) {
+ if (n.startsWith(DIGEST_KEY_PREFIX)) {
+ digestA.add(n);
+ }
+ }
+ }
+ Metadata bMain = metadataListB.get(0);
+ for (String n : bMain.names()) {
+ if (digestA.contains(n)) {
+ return n;
+ }
+ }
+ return null;
+ }
+
private void writeEmbeddedFilePathData(int i, String fileId, Metadata mA, Metadata mB) {
//container file, don't write anything
if (i == 0) {
@@ -411,12 +447,12 @@ public class ExtractComparer extends AbstractProfiler {
* Try to find the matching metadata based on the AbstractRecursiveParserWrapperHandler.EMBEDDED_RESOURCE_PATH
* If you can't find it, return -1;
*
- * @param i index for match in metadataListA
+ * @param aIndex index for match in metadataListA
* @param metadataListA
* @param metadataListB
* @return
*/
- private int getMatch(int i,
+ private int getMatch(int aIndex, String sharedDigestKey, Set<Integer> handledB,
List<Metadata> metadataListA,
List<Metadata> metadataListB) {
//TODO: could make this more robust
@@ -424,19 +460,19 @@ public class ExtractComparer extends AbstractProfiler {
return -1;
}
//assume first is always the container file
- if (i == 0) {
+ if (aIndex == 0) {
return 0;
}
- //first try to find matching digests
- //this does not elegantly handle multiple matching digests
- int match = findMatchingDigests(metadataListA.get(i), metadataListB);
- if (match > -1) {
- return match;
+ if (sharedDigestKey != null) {
+ //first try to find matching digests
+ //this does not elegantly handle multiple matching digests
+ return findMatchingDigests(sharedDigestKey, handledB,
+ metadataListA.get(aIndex), metadataListB);
}
//assume same embedded resource path. Not always true!
- Metadata thisMetadata = metadataListA.get(i);
+ Metadata thisMetadata = metadataListA.get(aIndex);
String embeddedPath = thisMetadata.get(AbstractRecursiveParserWrapperHandler.EMBEDDED_RESOURCE_PATH);
if (embeddedPath != null) {
for (int j = 0; j < metadataListB.size(); j++) {
@@ -451,22 +487,27 @@ public class ExtractComparer extends AbstractProfiler {
//last resort, if lists are same size, guess the same index
if (metadataListA.size() == metadataListB.size()) {
//assume no rearrangments if lists are the same size
- return i;
+ return aIndex;
}
return -1;
}
- private int findMatchingDigests(Metadata metadata, List<Metadata> metadataListB) {
- Set<String> digestKeys = new HashSet<>();
- for (String n : metadata.names()) {
- if (n.startsWith(DIGEST_KEY_PREFIX)) {
- String digestA = metadata.get(n);
- for (int i = 0; i < metadataListB.size(); i++) {
- String digestB = metadataListB.get(i).get(n);
- if (digestA != null && digestA.equals(digestB)) {
- return i;
- }
- }
+ private int findMatchingDigests(String sharedDigestKey,
+ Set<Integer> handledB,
+ Metadata metadata, List<Metadata> metadataListB) {
+ String digestA = metadata.get(sharedDigestKey);
+ if (digestA == null) {
+ return -1;
+ }
+
+ for (int i = 0; i < metadataListB.size(); i++) {
+ if (handledB.contains(i)) {
+ continue;
+ }
+ Metadata mB = metadataListB.get(i);
+ String digestB = mB.get(sharedDigestKey);
+ if (digestA.equalsIgnoreCase(digestB)) {
+ return i;
}
}
return -1;
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java
index 3eb0f23..27c335a 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/ContentLengthCalculator.java
@@ -19,6 +19,6 @@ package org.apache.tika.eval.textstats;
public class ContentLengthCalculator implements StringStatsCalculator<Integer> {
@Override
public Integer calculate(String txt) {
- return txt.length();
+ return txt.trim().length();
}
}
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
index 78b8b42..bae5792 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/SimpleComparerTest.java
@@ -281,7 +281,7 @@ public class SimpleComparerTest extends TikaTest {
List<Map<Cols, String>> tableInfos = WRITER.getTable(ExtractComparer.CONTENT_COMPARISONS);
assertEquals(3, tableInfos.size());
for (int i = 0; i < tableInfos.size(); i++) {
- assertEquals("1.0", tableInfos.get(i).get(Cols.OVERLAP));
+ assertEquals("problem with "+i, "1.0", tableInfos.get(i).get(Cols.OVERLAP));
}
}
@@ -439,6 +439,5 @@ public class SimpleComparerTest extends TikaTest {
ExtractComparer.CONTENT_COMPARISONS}) {
debugPrintTable(t);
}
-
}
}
diff --git a/tika-eval/src/test/resources/test-dirs/extractsA/file14_diffAttachOrder.json b/tika-eval/src/test/resources/test-dirs/extractsA/file14_diffAttachOrder.json
index fb16381..7801ab8 100644
--- a/tika-eval/src/test/resources/test-dirs/extractsA/file14_diffAttachOrder.json
+++ b/tika-eval/src/test/resources/test-dirs/extractsA/file14_diffAttachOrder.json
@@ -1,18 +1,19 @@
-[{
- "Content-Type":"text/plain",
- "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
-},
+[
{
- "Content-Type":"text/plain",
- "X-TIKA:embedded_resource_path":"/0",
- "X-TIKA:content":"a b c d",
- "X-TIKA:digest:MD5":"471d98383e9f40444e5ecf821f2c8354"
+ "Content-Type": "text/plain",
+ "X-TIKA:content": "the quick brown fox fox fox jumped over the lazy lazy dog",
+ "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8351"
},
{
- "Content-Type":"text/plain",
- "X-TIKA:embedded_resource_path":"/1",
- "X-TIKA:content":"e f g",
- "X-TIKA:digest:MD5":"471d98383e9f40444e5ecf821f2c8353"
+ "Content-Type": "text/plain",
+ "X-TIKA:embedded_resource_path": "/0",
+ "X-TIKA:content": "a b c d e f g h i j k l m n",
+ "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8354"
+ },
+ {
+ "Content-Type": "text/plain",
+ "X-TIKA:embedded_resource_path": "/1",
+ "X-TIKA:content": "o p q r s t u v w x y z",
+ "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8353"
}
-
]
\ No newline at end of file
diff --git a/tika-eval/src/test/resources/test-dirs/extractsB/file14_diffAttachOrder.json b/tika-eval/src/test/resources/test-dirs/extractsB/file14_diffAttachOrder.json
index edd0a69..e28bace 100644
--- a/tika-eval/src/test/resources/test-dirs/extractsB/file14_diffAttachOrder.json
+++ b/tika-eval/src/test/resources/test-dirs/extractsB/file14_diffAttachOrder.json
@@ -1,17 +1,19 @@
-[{
- "Content-Type":"text/plain",
- "X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog"
-},
+[
{
- "Content-Type":"text/plain",
- "X-TIKA:embedded_resource_path":"inner2.txt",
- "X-TIKA:content":"e f g",
- "X-TIKA:digest:MD5":"471d98383e9f40444e5ecf821f2c8353"
+ "Content-Type": "text/plain",
+ "X-TIKA:content": "the quick brown fox fox fox jumped over the lazy lazy dog",
+ "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8351"
},
{
- "Content-Type":"text/plain",
- "X-TIKA:embedded_resource_path":"inner1.txt",
- "X-TIKA:content":"a b c d",
- "X-TIKA:digest:MD5":"471d98383e9f40444e5ecf821f2c8354"
+ "Content-Type": "text/plain",
+ "X-TIKA:embedded_resource_path": "inner2.txt",
+ "X-TIKA:content": "o p q r s t u v w x y z",
+ "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8353"
+ },
+ {
+ "Content-Type": "text/plain",
+ "X-TIKA:embedded_resource_path": "inner1.txt",
+ "X-TIKA:content": "a b c d e f g h i j k l m n",
+ "X-TIKA:digest:MD5": "471d98383e9f40444e5ecf821f2c8354"
}
]
\ No newline at end of file
[tika] 02/04: TIKA-3061 -- need to read inputstream completely
before processing in StreamingZipContainerDetector
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 8a2cf1ec3e8f39ba40759db0d30343b178ebfede
Author: tallison <ta...@apache.org>
AuthorDate: Fri Mar 6 11:35:51 2020 -0500
TIKA-3061 -- need to read inputstream completely before processing in StreamingZipContainerDetector
---
tika-core/src/test/java/org/apache/tika/TikaTest.java | 5 +++++
.../tika/parser/pkg/StreamingZipContainerDetector.java | 10 +++++++++-
.../apache/tika/detect/TestContainerAwareDetector.java | 9 +++++++++
.../resources/test-documents/testOpenOfficeInAZip.zip | Bin 0 -> 10706 bytes
4 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/tika-core/src/test/java/org/apache/tika/TikaTest.java b/tika-core/src/test/java/org/apache/tika/TikaTest.java
index cbd8f1c..ac1ef3c 100644
--- a/tika-core/src/test/java/org/apache/tika/TikaTest.java
+++ b/tika-core/src/test/java/org/apache/tika/TikaTest.java
@@ -255,6 +255,11 @@ public abstract class TikaTest {
return getRecursiveMetadata(tis, new ParseContext(), new Metadata(), suppressException);
}
}
+ protected List<Metadata> getRecursiveMetadata(Path filePath) throws Exception {
+ try (TikaInputStream tis = TikaInputStream.get(filePath)) {
+ return getRecursiveMetadata(tis, true);
+ }
+ }
protected List<Metadata> getRecursiveMetadata(InputStream is, boolean suppressException) throws Exception {
return getRecursiveMetadata(is, new ParseContext(), new Metadata(), suppressException);
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java
index b55ed1a..a39f9f4 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/pkg/StreamingZipContainerDetector.java
@@ -18,6 +18,7 @@ package org.apache.tika.parser.pkg;
import static java.nio.charset.StandardCharsets.UTF_8;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.util.HashSet;
@@ -139,7 +140,13 @@ public class StreamingZipContainerDetector extends ZipContainerDetectorBase impl
return MediaType.parse(new String(bos.toByteArray(), UTF_8));
}
} else if (name.equals("META-INF/manifest.xml")) {
- MediaType mt = detectStarOfficeX(zipArchiveInputStream);
+ //for an unknown reason, passing in the zipArchiveInputStream
+ //"as is" can cause the iteration of the entries to stop early
+ //without exception or warning. So, copy the full stream, then
+ //process. TIKA-3061
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ IOUtils.copy(zipArchiveInputStream, bos);
+ MediaType mt = detectStarOfficeX(new ByteArrayInputStream(bos.toByteArray()));
if (mt != null) {
return mt;
}
@@ -157,6 +164,7 @@ public class StreamingZipContainerDetector extends ZipContainerDetectorBase impl
} catch (SecurityException e) {
throw e;
} catch (Exception e) {
+ e.printStackTrace();
//swallow
}
//entrynames is the union of directory names and file names
diff --git a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
index ed27456..2fa274a 100644
--- a/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
+++ b/tika-parsers/src/test/java/org/apache/tika/detect/TestContainerAwareDetector.java
@@ -25,6 +25,7 @@ import java.io.FileFilter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
import java.util.Random;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
@@ -554,4 +555,12 @@ public class TestContainerAwareDetector extends MultiThreadedTikaTest {
testDetector(detector, numThreads, 50, filter, numThreads*3);
}
+ @Test
+ public void testOpenOfficeInAZip() throws Exception {
+ List<Metadata> metadataList = getRecursiveMetadata(
+ "testOpenOfficeInAZip.zip");
+ assertEquals(3, metadataList.size());
+ assertEquals("application/vnd.oasis.opendocument.presentation",
+ metadataList.get(2).get(Metadata.CONTENT_TYPE));
+ }
}
diff --git a/tika-parsers/src/test/resources/test-documents/testOpenOfficeInAZip.zip b/tika-parsers/src/test/resources/test-documents/testOpenOfficeInAZip.zip
new file mode 100644
index 0000000..2546048
Binary files /dev/null and b/tika-parsers/src/test/resources/test-documents/testOpenOfficeInAZip.zip differ