You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2012/06/29 23:19:11 UTC
svn commit: r1355562 - in
/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg:
CompressorParser.java PackageParser.java ZipContainerDetector.java
Author: jukka
Date: Fri Jun 29 21:19:10 2012
New Revision: 1355562
URL: http://svn.apache.org/viewvc?rev=1355562&view=rev
Log:
TIKA-932: Upgrade to Commons Compress 1.4.1
Extend the ZipContainerDetector to detect also other compression and archive formats with the autodetection features in Commons Compress.
Modified:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java?rev=1355562&r1=1355561&r2=1355562&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java Fri Jun 29 21:19:10 2012
@@ -60,6 +60,20 @@ public class CompressorParser extends Ab
private static final Set<MediaType> SUPPORTED_TYPES =
MediaType.set(BZIP, BZIP2, GZIP, XZ, PACK);
+ static MediaType getMediaType(CompressorInputStream stream) {
+ if (stream instanceof BZip2CompressorInputStream) {
+ return BZIP2;
+ } else if (stream instanceof GzipCompressorInputStream) {
+ return GZIP;
+ } else if (stream instanceof XZCompressorInputStream) {
+ return XZ;
+ } else if (stream instanceof Pack200CompressorInputStream) {
+ return PACK;
+ } else {
+ return MediaType.OCTET_STREAM;
+ }
+ }
+
public Set<MediaType> getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
}
@@ -84,14 +98,9 @@ public class CompressorParser extends Ab
throw new TikaException("Unable to uncompress document stream", e);
}
- if (cis instanceof BZip2CompressorInputStream) {
- metadata.set(CONTENT_TYPE, BZIP2.toString());
- } else if (cis instanceof GzipCompressorInputStream) {
- metadata.set(CONTENT_TYPE, GZIP.toString());
- } else if (cis instanceof XZCompressorInputStream) {
- metadata.set(CONTENT_TYPE, XZ.toString());
- } else if (cis instanceof Pack200CompressorInputStream) {
- metadata.set(CONTENT_TYPE, PACK.toString());
+ MediaType type = getMediaType(cis);
+ if (!type.equals(MediaType.OCTET_STREAM)) {
+ metadata.set(CONTENT_TYPE, type.toString());
}
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageParser.java?rev=1355562&r1=1355561&r2=1355562&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageParser.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageParser.java Fri Jun 29 21:19:10 2012
@@ -68,6 +68,28 @@ public class PackageParser extends Abstr
private static final Set<MediaType> SUPPORTED_TYPES =
MediaType.set(ZIP, JAR, AR, CPIO, DUMP, TAR);
+ static MediaType getMediaType(ArchiveInputStream stream) {
+ if (stream instanceof JarArchiveInputStream) {
+ return JAR;
+ } else if (stream instanceof ZipArchiveInputStream) {
+ return ZIP;
+ } else if (stream instanceof ArArchiveInputStream) {
+ return AR;
+ } else if (stream instanceof CpioArchiveInputStream) {
+ return CPIO;
+ } else if (stream instanceof DumpArchiveInputStream) {
+ return DUMP;
+ } else if (stream instanceof TarArchiveInputStream) {
+ return TAR;
+ } else {
+ return MediaType.OCTET_STREAM;
+ }
+ }
+
+ static boolean isZipArchive(MediaType type) {
+ return type.equals(ZIP) || type.equals(JAR);
+ }
+
public Set<MediaType> getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
}
@@ -92,18 +114,9 @@ public class PackageParser extends Abstr
throw new TikaException("Unable to unpack document stream", e);
}
- if (ais instanceof JarArchiveInputStream) {
- metadata.set(CONTENT_TYPE, JAR.toString());
- } else if (ais instanceof ZipArchiveInputStream) {
- metadata.set(CONTENT_TYPE, ZIP.toString());
- } else if (ais instanceof ArArchiveInputStream) {
- metadata.set(CONTENT_TYPE, AR.toString());
- } else if (ais instanceof CpioArchiveInputStream) {
- metadata.set(CONTENT_TYPE, CPIO.toString());
- } else if (ais instanceof DumpArchiveInputStream) {
- metadata.set(CONTENT_TYPE, DUMP.toString());
- } else if (ais instanceof TarArchiveInputStream) {
- metadata.set(CONTENT_TYPE, TAR.toString());
+ MediaType type = getMediaType(ais);
+ if (!type.equals(MediaType.OCTET_STREAM)) {
+ metadata.set(CONTENT_TYPE, type.toString());
}
// Use the delegate parser to parse the contained document
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java?rev=1355562&r1=1355561&r2=1355562&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ZipContainerDetector.java Fri Jun 29 21:19:10 2012
@@ -16,12 +16,20 @@
*/
package org.apache.tika.parser.pkg;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Pattern;
+import org.apache.commons.compress.archivers.ArchiveException;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.ArchiveStreamFactory;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.poi.extractor.ExtractorFactory;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.opc.OPCPackage;
@@ -29,7 +37,9 @@ import org.apache.poi.openxml4j.opc.Pack
import org.apache.poi.openxml4j.opc.PackagePart;
import org.apache.poi.openxml4j.opc.PackageRelationshipCollection;
import org.apache.tika.detect.Detector;
+import org.apache.tika.exception.TikaException;
import org.apache.tika.io.IOUtils;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -37,8 +47,8 @@ import org.apache.tika.parser.iwork.IWor
import org.apache.tika.parser.iwork.IWorkPackageParser.IWORKDocumentType;
/**
- * A detector that works on a Zip document
- * to figure out exactly what the file is
+ * A detector that works on Zip documents and other archive and compression
+ * formats to figure out exactly what the file is.
*/
public class ZipContainerDetector implements Detector {
private static final Pattern MACRO_TEMPLATE_PATTERN = Pattern.compile("macroenabledtemplate$", Pattern.CASE_INSENSITIVE);
@@ -53,47 +63,98 @@ public class ZipContainerDetector implem
return MediaType.OCTET_STREAM;
}
- // Check if the document starts with the Zip header
- input.mark(4);
+ TemporaryResources tmp = new TemporaryResources();
try {
- if (input.read() != 'P' || input.read() != 'K'
- || input.read() != 3 || input.read() != 4) {
- return MediaType.OCTET_STREAM;
+ TikaInputStream tis = TikaInputStream.get(input, tmp);
+
+ byte[] prefix = new byte[1024]; // enough for all known formats
+ int length = tis.peek(prefix);
+
+ MediaType type = detectArchiveFormat(prefix, length);
+ if (PackageParser.isZipArchive(type)
+ && TikaInputStream.isTikaInputStream(input)) {
+ return detectZipFormat(tis);
+ } else if (!type.equals(MediaType.OCTET_STREAM)) {
+ return type;
+ } else {
+ return detectCompressorFormat(prefix, length);
}
} finally {
- input.reset();
+ try {
+ tmp.dispose();
+ } catch (TikaException e) {
+ // ignore
+ }
}
+ }
- // We can only detect the exact type when given a TikaInputStream
- TikaInputStream tis = TikaInputStream.cast(input);
- if (tis != null) {
+ private static MediaType detectCompressorFormat(byte[] prefix, int length) {
+ try {
+ CompressorStreamFactory factory = new CompressorStreamFactory();
+ CompressorInputStream cis = factory.createCompressorInputStream(
+ new ByteArrayInputStream(prefix, 0, length));
try {
- ZipFile zip = new ZipFile(tis.getFile());
+ return CompressorParser.getMediaType(cis);
+ } finally {
+ IOUtils.closeQuietly(cis);
+ }
+ } catch (CompressorException e) {
+ return MediaType.OCTET_STREAM;
+ }
+ }
+
+ private static MediaType detectArchiveFormat(byte[] prefix, int length) {
+ try {
+ ArchiveStreamFactory factory = new ArchiveStreamFactory();
+ ArchiveInputStream ais = factory.createArchiveInputStream(
+ new ByteArrayInputStream(prefix, 0, length));
+ try {
+ if ((ais instanceof TarArchiveInputStream)
+ && !TarArchiveInputStream.matches(prefix, length)) {
+ // ArchiveStreamFactory is too relaxed, see COMPRESS-117
+ return MediaType.OCTET_STREAM;
+ } else {
+ return PackageParser.getMediaType(ais);
+ }
+ } finally {
+ IOUtils.closeQuietly(ais);
+ }
+ } catch (ArchiveException e) {
+ return MediaType.OCTET_STREAM;
+ }
+ }
+
+ private static MediaType detectZipFormat(TikaInputStream tis) {
+ try {
+ ZipFile zip = new ZipFile(tis.getFile()); // TODO: hasFile()?
+ try {
+ MediaType type = detectOpenDocument(zip);
+ if (type == null) {
+ type = detectOfficeOpenXML(zip, tis);
+ }
+ if (type == null) {
+ type = detectIWork(zip);
+ }
+ if (type == null) {
+ type = detectJar(zip);
+ }
+ if (type != null) {
+ return type;
+ }
+ } finally {
+ // TODO: shouldn't we record the open
+ // container so it can be later
+ // reused...?
+ // tis.setOpenContainer(zip);
try {
- MediaType type = detectOpenDocument(zip);
- if (type == null) {
- type = detectOfficeOpenXML(zip, tis);
- }
- if (type == null) {
- type = detectIWork(zip);
- }
- if (type == null) {
- type = detectJar(zip);
- }
- if (type != null) {
- return type;
- }
- } finally {
- // TODO: shouldn't we record the open
- // container so it can be later
- // reused...?
- // tis.setOpenContainer(zip);
zip.close();
+ } catch (IOException e) {
+ // ignore
}
- } catch (IOException ignore) {
}
+ } catch (IOException e) {
+ // ignore
}
-
// Fallback: it's still a zip file, we just don't know what kind of one
return MediaType.APPLICATION_ZIP;
}