You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2015/04/10 02:58:35 UTC

svn commit: r1672528 - in /tika/trunk: ./ tika-parsers/ tika-parsers/src/main/appended-resources/META-INF/ tika-parsers/src/main/java/org/apache/tika/parser/image/ tika-parsers/src/main/resources/META-INF/services/ tika-parsers/src/test/java/org/apache...

Author: tallison
Date: Fri Apr 10 00:58:34 2015
New Revision: 1672528

URL: http://svn.apache.org/r1672528
Log:
TIKA-1594 upgrade metadata-extractor to 2.8.0 and add parser for webp metadata

Added:
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java
    tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java
    tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp   (with props)
    tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp   (with props)
Modified:
    tika/trunk/CHANGES.txt
    tika/trunk/tika-parsers/pom.xml
    tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
    tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser

Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Fri Apr 10 00:58:34 2015
@@ -1,3 +1,7 @@
+Release 1.9 - ???
+  * Upgrade to com.drewnoakes' metadata-extractor to 2.0 and
+    add parser for webp metadata (TIKA-1594).
+
 Release 1.8 - 4/7/2015
   * Duration extracted from MP3s with no ID3 tags (TIKA-1589).
 

Modified: tika/trunk/tika-parsers/pom.xml
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/pom.xml?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/pom.xml (original)
+++ tika/trunk/tika-parsers/pom.xml Fri Apr 10 00:58:34 2015
@@ -178,7 +178,7 @@
     <dependency>
       <groupId>com.drewnoakes</groupId>
       <artifactId>metadata-extractor</artifactId>
-      <version>2.7.2</version>
+      <version>2.8.0</version>
     </dependency>
     <dependency>
       <groupId>de.l3s.boilerpipe</groupId>

Modified: tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE (original)
+++ tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE Fri Apr 10 00:58:34 2015
@@ -82,4 +82,13 @@ JUnRAR (https://github.com/edmund-wagner
 
 Sqlite (bundled in org.xerial's sqlite-jdbc)
     This product bundles Sqlite, which is in the Public Domain.  For details
-    see: https://www.sqlite.org/copyright.html
\ No newline at end of file
+    see: https://www.sqlite.org/copyright.html
+
+Two photos in test-documents (testWebp_Alpha_Lossy.webp and testWebp_Alpha_Lossless.webp)
+    are in the public domain.  These files were retrieved from:
+    https://github.com/drewnoakes/metadata-extractor-images/tree/master/webp
+    These photos are also available here:
+    https://developers.google.com/speed/webp/gallery2#webp_links
+    Credits for the photo:
+    "Free Stock Photo in High Resolution - Yellow Rose 3 - Flowers"
+    Image Author: Jon Sullivan

Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java Fri Apr 10 00:58:34 2015
@@ -28,19 +28,13 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.poi.util.IOUtils;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.IPTC;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.xml.sax.SAXException;
-
 import com.drew.imaging.jpeg.JpegMetadataReader;
 import com.drew.imaging.jpeg.JpegProcessingException;
-import com.drew.imaging.jpeg.JpegSegmentType;
+import com.drew.imaging.riff.RiffProcessingException;
 import com.drew.imaging.tiff.TiffMetadataReader;
 import com.drew.imaging.tiff.TiffProcessingException;
+import com.drew.imaging.webp.WebpMetadataReader;
+import com.drew.lang.ByteArrayReader;
 import com.drew.lang.GeoLocation;
 import com.drew.lang.Rational;
 import com.drew.metadata.Directory;
@@ -55,12 +49,20 @@ import com.drew.metadata.iptc.IptcDirect
 import com.drew.metadata.jpeg.JpegCommentDirectory;
 import com.drew.metadata.jpeg.JpegDirectory;
 import com.drew.metadata.xmp.XmpReader;
+import org.apache.poi.util.IOUtils;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.IPTC;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.xml.sax.SAXException;
 
 /**
  * Uses the <a href="http://www.drewnoakes.com/code/exif/">Metadata Extractor</a> library
  * to read EXIF and IPTC image metadata and map to Tika fields.
  * <p/>
  * As of 2.4.0 the library supports jpeg and tiff.
+ * As of 2.8.0 the library supports webp.
  */
 public class ImageMetadataExtractor {
 
@@ -115,6 +117,21 @@ public class ImageMetadataExtractor {
         }
     }
 
+    public void parseWebP(File file) throws IOException, TikaException {
+
+        try {
+            com.drew.metadata.Metadata webPMetadata = new com.drew.metadata.Metadata();
+            webPMetadata = WebpMetadataReader.readMetadata(file);
+            handle(webPMetadata);
+        } catch (IOException e) {
+            throw e;
+        } catch (RiffProcessingException e) {
+            throw new TikaException("Can't process Riff data", e);
+        } catch (MetadataException e) {
+            throw new TikaException("Can't process Riff data", e);
+        }
+    }
+
     public void parseRawExif(InputStream stream, int length, boolean needsExifHeader)
             throws IOException, SAXException, TikaException {
         byte[] exif;
@@ -136,7 +153,7 @@ public class ImageMetadataExtractor {
             throws IOException, SAXException, TikaException {
         com.drew.metadata.Metadata metadata = new com.drew.metadata.Metadata();
         ExifReader reader = new ExifReader();
-        reader.extract(exifData, metadata, JpegSegmentType.APP1);
+        reader.extract(new ByteArrayReader(exifData), metadata, ExifReader.JPEG_SEGMENT_PREAMBLE.length());
 
         try {
             handle(metadata);
@@ -186,6 +203,15 @@ public class ImageMetadataExtractor {
         }
     }
 
+    private static String trimPixels(String s) {
+        //if height/width appears as "100 pixels", trim " pixels"
+        if (s != null) {
+            int i = s.lastIndexOf(" pixels");
+            s = s.substring(0, i);
+        }
+        return s;
+    }
+
     /**
      * Reads one or more type of Metadata Extractor fields.
      */
@@ -271,9 +297,7 @@ public class ImageMetadataExtractor {
             //Exif.Image.ImageWidth                        Short       1  100
             //Exif.Image.ImageLength                       Short       1  75
             // and the values are found in "Thumbnail Image Width" (and Height) from Metadata Extractor
-            set(directory, metadata, ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_WIDTH, Metadata.IMAGE_WIDTH);
             set(directory, metadata, JpegDirectory.TAG_IMAGE_WIDTH, Metadata.IMAGE_WIDTH);
-            set(directory, metadata, ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_HEIGHT, Metadata.IMAGE_LENGTH);
             set(directory, metadata, JpegDirectory.TAG_IMAGE_HEIGHT, Metadata.IMAGE_LENGTH);
             // Bits per sample, two methods of extracting, exif overrides jpeg
             set(directory, metadata, JpegDirectory.TAG_DATA_PRECISION, Metadata.BITS_PER_SAMPLE);
@@ -426,11 +450,13 @@ public class ImageMetadataExtractor {
             if (directory.containsTag(ExifIFD0Directory.TAG_RESOLUTION_UNIT)) {
                 metadata.set(Metadata.RESOLUTION_UNIT, directory.getDescription(ExifIFD0Directory.TAG_RESOLUTION_UNIT));
             }
-            if (directory.containsTag(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_WIDTH)) {
-                metadata.set(Metadata.IMAGE_WIDTH, directory.getDescription(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_WIDTH));
-            }
-            if (directory.containsTag(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_HEIGHT)) {
-                metadata.set(Metadata.IMAGE_LENGTH, directory.getDescription(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_HEIGHT));
+            if (directory.containsTag(ExifThumbnailDirectory.TAG_IMAGE_WIDTH)) {
+                        metadata.set(Metadata.IMAGE_WIDTH,
+                                trimPixels(directory.getDescription(ExifThumbnailDirectory.TAG_IMAGE_WIDTH)));
+            }
+            if (directory.containsTag(ExifThumbnailDirectory.TAG_IMAGE_HEIGHT)) {
+                metadata.set(Metadata.IMAGE_LENGTH,
+                        trimPixels(directory.getDescription(ExifThumbnailDirectory.TAG_IMAGE_HEIGHT)));
             }
         }
 

Added: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java?rev=1672528&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java (added)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java Fri Apr 10 00:58:34 2015
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+
+public class WebPParser extends AbstractParser {
+
+    /** Serial version UID */
+    private static final long serialVersionUID = -3941143576535464926L;
+
+    private static final Set<MediaType> SUPPORTED_TYPES =
+            Collections.singleton(MediaType.image("webp"));
+
+    public Set<MediaType> getSupportedTypes(ParseContext context) {
+        return SUPPORTED_TYPES;
+    }
+
+    public void parse(
+            InputStream stream, ContentHandler handler,
+            Metadata metadata, ParseContext context)
+            throws IOException, SAXException, TikaException {
+        TemporaryResources tmp = new TemporaryResources();
+        try {
+            TikaInputStream tis = TikaInputStream.get(stream, tmp);
+            new ImageMetadataExtractor(metadata).parseWebP(tis.getFile());
+        } finally {
+            tmp.dispose();
+        }
+
+        XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+        xhtml.startDocument();
+        xhtml.endDocument();
+    }
+}

Modified: tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser (original)
+++ tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser Fri Apr 10 00:58:34 2015
@@ -28,6 +28,7 @@ org.apache.tika.parser.image.BPGParser
 org.apache.tika.parser.image.ImageParser
 org.apache.tika.parser.image.PSDParser
 org.apache.tika.parser.image.TiffParser
+org.apache.tika.parser.image.WebPParser
 org.apache.tika.parser.iptc.IptcAnpaParser
 org.apache.tika.parser.iwork.IWorkPackageParser
 org.apache.tika.parser.jpeg.JpegParser

Added: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java?rev=1672528&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java (added)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java Fri Apr 10 00:58:34 2015
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.image;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.InputStream;
+
+import org.apache.tika.io.IOUtils;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.junit.Test;
+import org.xml.sax.helpers.DefaultHandler;
+
+
+public class WebPParserTest {
+
+    Parser parser = new AutoDetectParser();
+/*
+    Two photos in test-documents (testWebp_Alpha_Lossy.webp and testWebp_Alpha_Lossless.webp)
+    are in the public domain.  These files were retrieved from:
+    https://github.com/drewnoakes/metadata-extractor-images/tree/master/webp
+    These photos are also available here:
+    https://developers.google.com/speed/webp/gallery2#webp_links
+    Credits for the photo:
+    "Free Stock Photo in High Resolution - Yellow Rose 3 - Flowers"
+    Image Author: Jon Sullivan
+ */
+    @Test
+    public void testSimple() throws Exception {
+        Metadata metadata = new Metadata();
+        InputStream stream =
+                getClass().getResourceAsStream("/test-documents/testWebp_Alpha_Lossy.webp");
+
+        parser.parse(stream, new DefaultHandler(), metadata, new ParseContext());
+
+        assertEquals("301", metadata.get("Image Height"));
+        assertEquals("400", metadata.get("Image Width"));
+        assertEquals("true", metadata.get("Has Alpha"));
+        assertEquals("false", metadata.get("Is Animation"));
+        assertEquals("image/webp", metadata.get(Metadata.CONTENT_TYPE));
+
+        IOUtils.closeQuietly(stream);
+
+        metadata = new Metadata();
+        stream = getClass().getResourceAsStream("/test-documents/testWebp_Alpha_Lossless.webp");
+        parser.parse(stream, new DefaultHandler(), metadata, new ParseContext());
+
+        //unfortunately, there isn't much metadata in lossless
+        assertEquals("image/webp", metadata.get(Metadata.CONTENT_TYPE));
+
+    }
+
+}

Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp?rev=1672528&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp?rev=1672528&view=auto
==============================================================================
Binary file - no diff available.

Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream