You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2010/05/26 12:41:49 UTC

svn commit: r948390 - in /tika/trunk/tika-parsers/src: main/java/org/apache/tika/parser/dwg/ test/java/org/apache/tika/parser/dwg/ test/resources/test-documents/

Author: jukka
Date: Wed May 26 10:41:48 2010
New Revision: 948390

URL: http://svn.apache.org/viewvc?rev=948390&view=rev
Log:
TIKA-413: DWG Parser

Contribution by Nick Burch

Added:
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java   (with props)
    tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/
    tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java   (with props)
    tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg   (with props)
    tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg   (with props)
    tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg   (with props)

Added: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java?rev=948390&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java (added)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java Wed May 26 10:41:48 2010
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.dwg;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.poi.util.IOUtils;
+import org.apache.poi.util.LittleEndian;
+import org.apache.poi.util.StringUtil;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+/**
+ * DWG (CAD Drawing) parser. This is a very basic parser, which just
+ *  looks for bits of the headers.
+ * Note that we use Apache POI for various parts of the processing, as
+ *  lots of the low level string/int/short concepts are the same.
+ */
+public class DWGParser implements Parser {
+    private static final Set<MediaType> SUPPORTED_TYPES =
+        Collections.unmodifiableSet(new HashSet<MediaType>(Arrays.asList(
+                MediaType.application("acad"),
+                MediaType.application("x-acad"),
+                MediaType.application("autocad_dwg"),
+                MediaType.application("dwg"),
+                MediaType.application("x-dwg"),
+                MediaType.application("x-autocad"),
+                MediaType.image("x-dwg"),
+                MediaType.image("vnd.dwg")
+        )));
+
+    public Set<MediaType> getSupportedTypes(ParseContext context) {
+        return SUPPORTED_TYPES;
+    }
+    
+    /** The order of the fields in the header */
+    private static String[] HEADER_PROPERTIES_ENTRIES = {
+        Metadata.TITLE, 
+        Metadata.SUBJECT,
+        Metadata.AUTHOR,
+        Metadata.KEYWORDS,
+        Metadata.COMMENTS,
+        Metadata.LAST_AUTHOR,
+        null, // Unknown?
+        Metadata.RELATION, // Hyperlink
+    };
+
+    public void parse(
+            InputStream stream, ContentHandler handler,
+            Metadata metadata, ParseContext context)
+            throws IOException, TikaException, SAXException {
+        
+        // First up, which version of the format are we handling?
+        byte[] header = new byte[128];
+        IOUtils.readFully(stream, header);
+        String version = new String(header, 0, 6);
+        
+        XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+        xhtml.startDocument();
+        
+        if(version.equals("AC1018")) {
+            skipToPropertyInfoSection(stream, header);
+            get2004Props(stream,metadata,xhtml);
+        } else if (version.equals("AC1021") || version.equals("AC1024")) {
+            skipToPropertyInfoSection(stream, header);
+            get2007and2010Props(stream,metadata,xhtml);
+        } else {
+            throw new IllegalArgumentException("Unsupported version '" + version + "' found");
+        }
+        
+        xhtml.endDocument();
+    }
+    
+    /**
+     * Stored as US-ASCII
+     */
+    private void get2004Props(InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) throws IOException, SAXException {
+        for(int i=0; i<HEADER_PROPERTIES_ENTRIES.length; i++) {
+            int stringLen = LittleEndian.readUShort(stream);
+            
+            byte[] stringData = new byte[stringLen];
+            IOUtils.readFully(stream, stringData);
+            
+            // Often but not always null terminated
+            if(stringData[stringLen-1] == 0) {
+                stringLen--;
+            }
+            String headerValue = StringUtil.getFromCompressedUnicode(stringData, 0, stringLen);
+            
+            handleHeader(i, headerValue, metadata, xhtml);
+        }
+    }
+    /**
+     * Stored as UCS2, so 16 bit "unicode"
+     */
+    private void get2007and2010Props(InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) throws IOException, SAXException {
+        for(int i=0; i<HEADER_PROPERTIES_ENTRIES.length; i++) {
+            int stringLen = LittleEndian.readUShort(stream);
+            
+            byte[] stringData = new byte[stringLen*2];
+            IOUtils.readFully(stream, stringData);
+            String headerValue = StringUtil.getFromUnicodeLE(stringData);
+            
+            handleHeader(i, headerValue, metadata, xhtml);
+        }
+    }
+    
+    private void handleHeader(int headerNumber, String value, Metadata metadata, XHTMLContentHandler xhtml) throws SAXException {
+        if(value == null || value.length() == 0) {
+            return;
+        }
+        
+        // Some strings are null terminated
+        if(value.charAt(value.length()-1) == 0) {
+            value = value.substring(0, value.length()-1);
+        }
+        
+        String headerProp = HEADER_PROPERTIES_ENTRIES[headerNumber];
+        if(headerProp != null) {
+            metadata.set(headerProp, value);
+        }
+       
+        xhtml.element("p", value);
+    }
+    
+    private void skipToPropertyInfoSection(InputStream stream, byte[] header) throws IOException {
+        // The offset is stored in the header from 0x20 onwards
+        long offsetToSection = LittleEndian.getLong(header, 0x20);
+        long toSkip = offsetToSection - header.length;
+        while(toSkip > 0) {
+            byte[] skip = new byte[Math.min((int)toSkip, 0x4000)];
+            IOUtils.readFully(stream, skip);
+            toSkip -= skip.length;
+        }
+    }
+
+    public void parse(
+            InputStream stream, ContentHandler handler, Metadata metadata)
+            throws IOException, SAXException, TikaException {
+        parse(stream, handler, metadata, new ParseContext());
+    }
+
+}

Propchange: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/dwg/DWGParser.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java?rev=948390&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java (added)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java Wed May 26 10:41:48 2010
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.dwg;
+
+import java.io.InputStream;
+
+import junit.framework.TestCase;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+
+public class DWGParserTest extends TestCase {
+    public void testDWG2004Parser() throws Exception {
+        InputStream input = DWGParserTest.class.getResourceAsStream(
+                "/test-documents/testDWG2004.dwg");
+        testParser(input);
+    }
+    public void testDWG2007Parser() throws Exception {
+        InputStream input = DWGParserTest.class.getResourceAsStream(
+                "/test-documents/testDWG2007.dwg");
+        testParser(input);
+    }
+    public void testDWG2010Parser() throws Exception {
+        InputStream input = DWGParserTest.class.getResourceAsStream(
+                "/test-documents/testDWG2010.dwg");
+        testParser(input);
+    }
+    
+    private void testParser(InputStream input) throws Exception {
+        try {
+            Metadata metadata = new Metadata();
+            ContentHandler handler = new BodyContentHandler();
+            new DWGParser().parse(input, handler, metadata);
+
+//            assertEquals(
+//                    "application/dwg",
+//                    metadata.get(Metadata.CONTENT_TYPE));
+            
+            assertEquals("The quick brown fox jumps over the lazy dog", 
+                    metadata.get(Metadata.TITLE));
+            assertEquals("Gym class featuring a brown fox and lazy dog",
+                    metadata.get(Metadata.SUBJECT));
+            assertEquals("Nevin Nollop",
+                    metadata.get(Metadata.AUTHOR));
+            assertEquals("Pangram, fox, dog",
+                    metadata.get(Metadata.KEYWORDS));
+            assertEquals("Lorem ipsum",
+                    metadata.get(Metadata.COMMENTS).substring(0,11));
+            assertEquals("http://www.alfresco.com",
+                    metadata.get(Metadata.RELATION));
+            
+            String content = handler.toString();
+            assertTrue(content.contains("The quick brown fox jumps over the lazy dog"));
+            assertTrue(content.contains("Gym class"));
+            assertTrue(content.contains("www.alfresco.com"));
+        } finally {
+            input.close();
+        }
+    }
+
+}

Propchange: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/dwg/DWGParserTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg?rev=948390&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2004.dwg
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg?rev=948390&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2007.dwg
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg?rev=948390&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testDWG2010.dwg
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream