You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2010/09/10 23:51:36 UTC
svn commit: r995992 - in /tika/trunk:
tika-core/src/main/java/org/apache/tika/extractor/
tika-parsers/src/main/java/org/apache/tika/parser/microsoft/
tika-parsers/src/main/java/org/apache/tika/parser/pkg/
Author: jukka
Date: Fri Sep 10 21:51:35 2010
New Revision: 995992
URL: http://svn.apache.org/viewvc?rev=995992&view=rev
Log:
TIKA-509: Container contents extraction
Move embedded document extraction functionality to a shared utility class, and implement support for selective extraction based on DocumentSelectors or FilenameFilters.
Added:
tika/trunk/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractor.java (with props)
Modified:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageExtractor.java
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractor.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractor.java?rev=995992&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractor.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractor.java Fri Sep 10 21:51:35 2010
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.extractor;
+
+import static org.apache.tika.sax.XHTMLContentHandler.XHTML;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.DocumentSelector;
+import org.apache.tika.io.CloseShieldInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.DelegatingParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.apache.tika.sax.EmbeddedContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
+
+/**
+ * Helper class for parsers of package archives or other compound document
+ * formats that support embedded or attached component documents.
+ *
+ * @since Apache Tika 0.8
+ */
+public class EmbeddedDocumentExtractor {
+
+ private static final File ABSTRACT_PATH = new File("");
+
+ private static final Parser DELEGATING_PARSER = new DelegatingParser();
+
+ private final ParseContext context;
+
+ public EmbeddedDocumentExtractor(ParseContext context) {
+ this.context = context;
+ }
+
+ public boolean shouldParseEmbedded(Metadata metadata) {
+ DocumentSelector selector = context.get(DocumentSelector.class);
+ if (selector != null) {
+ return selector.select(metadata);
+ }
+
+ FilenameFilter filter = context.get(FilenameFilter.class);
+ if (filter != null) {
+ String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
+ if (name != null) {
+ return filter.accept(ABSTRACT_PATH, name);
+ }
+ }
+
+ return true;
+ }
+
+ public void parseEmbedded(
+ InputStream stream, ContentHandler handler, Metadata metadata)
+ throws SAXException, IOException {
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", "class", "class", "CDATA", "package-entry");
+ handler.startElement(XHTML, "div", "div", attributes);
+
+ String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
+ if (name != null && name.length() > 0) {
+ handler.startElement(XHTML, "h1", "h1", new AttributesImpl());
+ char[] chars = name.toCharArray();
+ handler.characters(chars, 0, chars.length);
+ handler.endElement(XHTML, "h1", "h1");
+ }
+
+ // Use the delegate parser to parse this entry
+ try {
+ DELEGATING_PARSER.parse(
+ new CloseShieldInputStream(stream),
+ new EmbeddedContentHandler(new BodyContentHandler(handler)),
+ metadata, context);
+ } catch (TikaException e) {
+ // Could not parse the entry, just skip the content
+ }
+
+ handler.endElement(XHTML, "div", "div");
+ }
+
+}
Propchange: tika/trunk/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentExtractor.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java?rev=995992&r1=995991&r2=995992&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java Fri Sep 10 21:51:35 2010
@@ -29,6 +29,7 @@ import org.apache.poi.poifs.filesystem.E
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.tika.detect.ZipContainerDetector;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -42,10 +43,10 @@ import org.xml.sax.SAXException;
abstract class AbstractPOIFSExtractor {
- private final ParseContext context;
+ private final EmbeddedDocumentExtractor extractor;
protected AbstractPOIFSExtractor(ParseContext context) {
- this.context = context;
+ this.extractor = new EmbeddedDocumentExtractor(context);
}
protected void handleEmbeddedResource(TikaInputStream resource,
@@ -55,16 +56,15 @@ abstract class AbstractPOIFSExtractor {
Metadata metadata = new Metadata();
if(filename != null) {
metadata.set(Metadata.TIKA_MIME_FILE, filename);
+ metadata.set(Metadata.RESOURCE_NAME_KEY, filename);
}
if(mediaType != null) {
metadata.set(Metadata.CONTENT_TYPE, mediaType);
}
-
- Parser parser = context.get(Parser.class, EmptyParser.INSTANCE);
- parser.parse(
- resource, new EmbeddedContentHandler(xhtml),
- metadata, context
- );
+
+ if (extractor.shouldParseEmbedded(metadata)) {
+ extractor.parseEmbedded(resource, xhtml, metadata);
+ }
} finally {
resource.close();
}
@@ -76,10 +76,10 @@ abstract class AbstractPOIFSExtractor {
protected void handleEmbededOfficeDoc(
DirectoryEntry dir, XHTMLContentHandler xhtml)
throws IOException, SAXException, TikaException {
- // Is it an embeded OLE2 document, or an embeded OOXML document?
+ // Is it an embedded OLE2 document, or an embedded OOXML document?
try {
Entry ooxml = dir.getEntry("Package");
-
+
// It's OOXML
TikaInputStream ooxmlStream = TikaInputStream.get(
new DocumentInputStream((DocumentEntry)ooxml)
@@ -91,7 +91,7 @@ abstract class AbstractPOIFSExtractor {
} catch(FileNotFoundException e) {
// It's regular OLE2
}
-
+
// Need to dump the directory out to a new temp file, so
// it's stand along
POIFSFileSystem newFS = new POIFSFileSystem();
@@ -109,14 +109,13 @@ abstract class AbstractPOIFSExtractor {
metadata.set(Metadata.CONTENT_TYPE, type.getType().toString());
// Trigger for the document itself
- TikaInputStream embeded = TikaInputStream.get(tmpFile);
+ TikaInputStream embedded = TikaInputStream.get(tmpFile);
try {
- Parser parser = context.get(Parser.class, EmptyParser.INSTANCE);
- parser.parse(
- embeded, new EmbeddedContentHandler(xhtml),
- metadata, context);
+ if (extractor.shouldParseEmbedded(metadata)) {
+ extractor.parseEmbedded(embedded, xhtml, metadata);
+ }
} finally {
- embeded.close();
+ embedded.close();
}
} finally {
tmpFile.delete();
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageExtractor.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageExtractor.java?rev=995992&r1=995991&r2=995992&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageExtractor.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/PackageExtractor.java Fri Sep 10 21:51:35 2010
@@ -30,13 +30,10 @@ import org.apache.commons.compress.archi
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipUtils;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.io.CloseShieldInputStream;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.EmptyParser;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -50,16 +47,13 @@ class PackageExtractor {
private final Metadata metadata;
- private final ParseContext context;
-
- private final Parser parser;
+ private final EmbeddedDocumentExtractor extractor;
public PackageExtractor(
ContentHandler handler, Metadata metadata, ParseContext context) {
this.handler = handler;
this.metadata = metadata;
- this.context = context;
- this.parser = context.get(Parser.class, EmptyParser.INSTANCE);
+ this.extractor = new EmbeddedDocumentExtractor(context);
}
public void parse(InputStream stream)
@@ -126,12 +120,11 @@ class PackageExtractor {
}
entrydata.set(Metadata.RESOURCE_NAME_KEY, name);
}
+
// Use the delegate parser to parse the compressed document
- parser.parse(
- new CloseShieldInputStream(stream),
- new EmbeddedContentHandler(
- new BodyContentHandler(xhtml)),
- entrydata, context);
+ if (extractor.shouldParseEmbedded(entrydata)) {
+ extractor.parseEmbedded(stream, xhtml, entrydata);
+ }
} finally {
stream.close();
}
@@ -155,24 +148,15 @@ class PackageExtractor {
ArchiveEntry entry = archive.getNextEntry();
while (entry != null) {
if (!entry.isDirectory()) {
- xhtml.startElement("div", "class", "package-entry");
Metadata entrydata = new Metadata();
String name = entry.getName();
if (name != null && name.length() > 0) {
entrydata.set(Metadata.RESOURCE_NAME_KEY, name);
- xhtml.element("h1", name);
}
- try {
- // Use the delegate parser to parse this entry
- parser.parse(
- new CloseShieldInputStream(archive),
- new EmbeddedContentHandler(
- new BodyContentHandler(xhtml)),
- entrydata, context);
- } catch (TikaException e) {
- // Could not parse the entry, just skip the content
+
+ if (extractor.shouldParseEmbedded(entrydata)) {
+ extractor.parseEmbedded(archive, xhtml, entrydata);
}
- xhtml.endElement("div");
}
entry = archive.getNextEntry();
}