You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ma...@apache.org on 2010/10/31 22:59:27 UTC
svn commit: r1029501 - in /tika/trunk: ./
tika-parsers/src/main/java/org/apache/tika/parser/hdf/
tika-parsers/src/test/java/org/apache/tika/parser/hdf/
tika-parsers/src/test/resources/test-documents/
Author: mattmann
Date: Sun Oct 31 21:59:26 2010
New Revision: 1029501
URL: http://svn.apache.org/viewvc?rev=1029501&view=rev
Log:
- fix for TIKA-399 HDF4/5 Tika Parser
Added:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/HDFParser.java (with props)
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/test.he5 (with props)
Modified:
tika/trunk/CHANGES.txt
Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1029501&r1=1029500&r2=1029501&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Sun Oct 31 21:59:26 2010
@@ -15,7 +15,7 @@ The most notable changes in Tika 0.8 ove
selecting the best parser for a given input document. (TIKA-298)
* Support for parsing common scientific data formats including netCDF
- was added (TIKA-400).
+ and HDF4/5 was added (TIKA-400 and TIKA-399).
* Unit tests for Windows have been fixed, allowing TestParsers
to complete. (TIKA-398)
Added: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/HDFParser.java?rev=1029501&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/HDFParser.java (added)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/HDFParser.java Sun Oct 31 21:59:26 2010
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.hdf;
+
+//JDK imports
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Set;
+
+//TIKA imports
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.netcdf.NetCDFParser;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+//NetCDF imports
+import ucar.nc2.Attribute;
+import ucar.nc2.Group;
+import ucar.nc2.NetcdfFile;
+
+/**
+ *
+ * Since the {@link NetCDFParser} depends on the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf-java" >NetCDF-Java</a> API,
+ * we are able to use it to parse HDF files as well. See <a href=
+ * "http://www.unidata.ucar.edu/software/netcdf-java/formats/FileTypes.html"
+ * >this link</a> for more information.
+ *
+ */
+public class HDFParser implements Parser {
+
+ private static final Set<MediaType> SUPPORTED_TYPES = Collections
+ .singleton(MediaType.application("application/x-hdf"));
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.tika.parser.netcdf.NetCDFParser#getSupportedTypes(org.apache
+ * .tika.parser.ParseContext)
+ */
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.tika.parser.netcdf.NetCDFParser#parse(java.io.InputStream,
+ * org.xml.sax.ContentHandler, org.apache.tika.metadata.Metadata,
+ * org.apache.tika.parser.ParseContext)
+ */
+ public void parse(InputStream stream, ContentHandler handler,
+ Metadata metadata, ParseContext context) throws IOException,
+ SAXException, TikaException {
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ this.writeStreamToMemory(stream, os);
+
+ NetcdfFile ncFile = NetcdfFile.openInMemory("", os.toByteArray());
+ this.unravelStringMet(ncFile, null, metadata);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.tika.parser.netcdf.NetCDFParser#parse(java.io.InputStream,
+ * org.xml.sax.ContentHandler, org.apache.tika.metadata.Metadata)
+ */
+ public void parse(InputStream stream, ContentHandler handler,
+ Metadata metadata) throws IOException, SAXException, TikaException {
+ this.parse(stream, handler, metadata, new ParseContext());
+ }
+
+ protected void unravelStringMet(NetcdfFile ncFile, Group group, Metadata met) {
+ if (group == null) {
+ group = ncFile.getRootGroup();
+ }
+
+ // unravel its string attrs
+ for (Attribute attribute : group.getAttributes()) {
+ if (attribute.isString()) {
+ met.add(attribute.getName(), attribute.getStringValue());
+ } else {
+ // try and cast its value to a string
+ met.add(attribute.getName(), String.valueOf(attribute
+ .getNumericValue()));
+ }
+ }
+
+ for (Group g : group.getGroups()) {
+ unravelStringMet(ncFile, g, met);
+ }
+ }
+
+ protected void writeStreamToMemory(InputStream is, ByteArrayOutputStream os)
+ throws TikaException {
+ byte[] buf = new byte[512];
+
+ try {
+ while ((is.read(buf, 0, 512)) != -1) {
+ os.write(buf);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new TikaException(e.getMessage());
+ }
+ }
+
+}
Propchange: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/hdf/HDFParser.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java?rev=1029501&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java (added)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java Sun Oct 31 21:59:26 2010
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.hdf;
+
+//JDK imports
+import java.io.InputStream;
+
+//TIKA imports
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.hdf.HDFParser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+
+//Junit imports
+import junit.framework.TestCase;
+
+/**
+ *
+ * Test suite for the {@link HDFParser}.
+ *
+ */
+public class HDFParserTest extends TestCase {
+
+ public void testParseGlobalMetadata() throws Exception {
+ Parser parser = new HDFParser();
+ ContentHandler handler = new BodyContentHandler();
+ Metadata metadata = new Metadata();
+
+ /*
+ * this is a publicly available HDF5 file from the MLS mission:
+ *
+ *
+ * ftp://acdisc.gsfc.nasa.gov/data/s4pa///Aura_MLS_Level2/ML2O3.002//2009
+ * /MLS-Aura_L2GP-O3_v02-23-c01_2009d122.he5
+ */
+ InputStream stream = HDFParser.class
+ .getResourceAsStream("/test-documents/test.he5");
+ try {
+ parser.parse(stream, handler, metadata);
+ } finally {
+ stream.close();
+ }
+
+ assertNotNull(metadata);
+ assertEquals("5", metadata.get("GranuleMonth"));
+ }
+
+}
Propchange: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/hdf/HDFParserTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/test.he5
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/test.he5?rev=1029501&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/test.he5
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream