You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2010/05/26 12:41:49 UTC
svn commit: r948390 - in /tika/trunk/tika-parsers/src:
main/java/org/apache/tika/parser/dwg/ test/java/org/apache/tika/parser/dwg/
test/resources/test-documents/
Author: jukka
Date: Wed May 26 10:41:48 2010
New Revision: 948390
URL: http://svn.apache.org/viewvc?rev=948390&view=rev
Log:
TIKA-413: DWG Parser
Contribution by Nick Burch
Added:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java (with props)
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg (with props)
Added: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java?rev=948390&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java (added)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java Wed May 26 10:41:48 2010
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.dwg;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.poi.util.IOUtils;
+import org.apache.poi.util.LittleEndian;
+import org.apache.poi.util.StringUtil;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+/**
+ * DWG (CAD Drawing) parser. This is a very basic parser, which just
+ * looks for bits of the headers.
+ * Note that we use Apache POI for various parts of the processing, as
+ * lots of the low level string/int/short concepts are the same.
+ */
+public class DWGParser implements Parser {
+ private static final Set<MediaType> SUPPORTED_TYPES =
+ Collections.unmodifiableSet(new HashSet<MediaType>(Arrays.asList(
+ MediaType.application("acad"),
+ MediaType.application("x-acad"),
+ MediaType.application("autocad_dwg"),
+ MediaType.application("dwg"),
+ MediaType.application("x-dwg"),
+ MediaType.application("x-autocad"),
+ MediaType.image("x-dwg"),
+ MediaType.image("vnd.dwg")
+ )));
+
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ /** The order of the fields in the header */
+ private static String[] HEADER_PROPERTIES_ENTRIES = {
+ Metadata.TITLE,
+ Metadata.SUBJECT,
+ Metadata.AUTHOR,
+ Metadata.KEYWORDS,
+ Metadata.COMMENTS,
+ Metadata.LAST_AUTHOR,
+ null, // Unknown?
+ Metadata.RELATION, // Hyperlink
+ };
+
+ public void parse(
+ InputStream stream, ContentHandler handler,
+ Metadata metadata, ParseContext context)
+ throws IOException, TikaException, SAXException {
+
+ // First up, which version of the format are we handling?
+ byte[] header = new byte[128];
+ IOUtils.readFully(stream, header);
+ String version = new String(header, 0, 6);
+
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+
+ if(version.equals("AC1018")) {
+ skipToPropertyInfoSection(stream, header);
+ get2004Props(stream,metadata,xhtml);
+ } else if (version.equals("AC1021") || version.equals("AC1024")) {
+ skipToPropertyInfoSection(stream, header);
+ get2007and2010Props(stream,metadata,xhtml);
+ } else {
+ throw new IllegalArgumentException("Unsupported version '" + version + "' found");
+ }
+
+ xhtml.endDocument();
+ }
+
+ /**
+ * Stored as US-ASCII
+ */
+ private void get2004Props(InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) throws IOException, SAXException {
+ for(int i=0; i<HEADER_PROPERTIES_ENTRIES.length; i++) {
+ int stringLen = LittleEndian.readUShort(stream);
+
+ byte[] stringData = new byte[stringLen];
+ IOUtils.readFully(stream, stringData);
+
+ // Often but not always null terminated
+ if(stringData[stringLen-1] == 0) {
+ stringLen--;
+ }
+ String headerValue = StringUtil.getFromCompressedUnicode(stringData, 0, stringLen);
+
+ handleHeader(i, headerValue, metadata, xhtml);
+ }
+ }
+ /**
+ * Stored as UCS2, so 16 bit "unicode"
+ */
+ private void get2007and2010Props(InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) throws IOException, SAXException {
+ for(int i=0; i<HEADER_PROPERTIES_ENTRIES.length; i++) {
+ int stringLen = LittleEndian.readUShort(stream);
+
+ byte[] stringData = new byte[stringLen*2];
+ IOUtils.readFully(stream, stringData);
+ String headerValue = StringUtil.getFromUnicodeLE(stringData);
+
+ handleHeader(i, headerValue, metadata, xhtml);
+ }
+ }
+
+ private void handleHeader(int headerNumber, String value, Metadata metadata, XHTMLContentHandler xhtml) throws SAXException {
+ if(value == null || value.length() == 0) {
+ return;
+ }
+
+ // Some strings are null terminated
+ if(value.charAt(value.length()-1) == 0) {
+ value = value.substring(0, value.length()-1);
+ }
+
+ String headerProp = HEADER_PROPERTIES_ENTRIES[headerNumber];
+ if(headerProp != null) {
+ metadata.set(headerProp, value);
+ }
+
+ xhtml.element("p", value);
+ }
+
+ private void skipToPropertyInfoSection(InputStream stream, byte[] header) throws IOException {
+ // The offset is stored in the header from 0x20 onwards
+ long offsetToSection = LittleEndian.getLong(header, 0x20);
+ long toSkip = offsetToSection - header.length;
+ while(toSkip > 0) {
+ byte[] skip = new byte[Math.min((int)toSkip, 0x4000)];
+ IOUtils.readFully(stream, skip);
+ toSkip -= skip.length;
+ }
+ }
+
+ public void parse(
+ InputStream stream, ContentHandler handler, Metadata metadata)
+ throws IOException, SAXException, TikaException {
+ parse(stream, handler, metadata, new ParseContext());
+ }
+
+}
Propchange: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java?rev=948390&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java (added)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java Wed May 26 10:41:48 2010
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.dwg;
+
+import java.io.InputStream;
+
+import junit.framework.TestCase;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+
+public class DWGParserTest extends TestCase {
+ public void testDWG2004Parser() throws Exception {
+ InputStream input = DWGParserTest.class.getResourceAsStream(
+ "/test-documents/testDWG2004.dwg");
+ testParser(input);
+ }
+ public void testDWG2007Parser() throws Exception {
+ InputStream input = DWGParserTest.class.getResourceAsStream(
+ "/test-documents/testDWG2007.dwg");
+ testParser(input);
+ }
+ public void testDWG2010Parser() throws Exception {
+ InputStream input = DWGParserTest.class.getResourceAsStream(
+ "/test-documents/testDWG2010.dwg");
+ testParser(input);
+ }
+
+ private void testParser(InputStream input) throws Exception {
+ try {
+ Metadata metadata = new Metadata();
+ ContentHandler handler = new BodyContentHandler();
+ new DWGParser().parse(input, handler, metadata);
+
+// assertEquals(
+// "application/dwg",
+// metadata.get(Metadata.CONTENT_TYPE));
+
+ assertEquals("The quick brown fox jumps over the lazy dog",
+ metadata.get(Metadata.TITLE));
+ assertEquals("Gym class featuring a brown fox and lazy dog",
+ metadata.get(Metadata.SUBJECT));
+ assertEquals("Nevin Nollop",
+ metadata.get(Metadata.AUTHOR));
+ assertEquals("Pangram, fox, dog",
+ metadata.get(Metadata.KEYWORDS));
+ assertEquals("Lorem ipsum",
+ metadata.get(Metadata.COMMENTS).substring(0,11));
+ assertEquals("http://www.alfresco.com",
+ metadata.get(Metadata.RELATION));
+
+ String content = handler.toString();
+ assertTrue(content.contains("The quick brown fox jumps over the lazy dog"));
+ assertTrue(content.contains("Gym class"));
+ assertTrue(content.contains("www.alfresco.com"));
+ } finally {
+ input.close();
+ }
+ }
+
+}
Propchange: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg?rev=948390&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg?rev=948390&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg?rev=948390&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream