You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2015/04/10 02:58:35 UTC
svn commit: r1672528 - in /tika/trunk: ./ tika-parsers/
tika-parsers/src/main/appended-resources/META-INF/
tika-parsers/src/main/java/org/apache/tika/parser/image/
tika-parsers/src/main/resources/META-INF/services/
tika-parsers/src/test/java/org/apache...
Author: tallison
Date: Fri Apr 10 00:58:34 2015
New Revision: 1672528
URL: http://svn.apache.org/r1672528
Log:
TIKA-1594 upgrade metadata-extractor to 2.8.0 and add parser for webp metadata
Added:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java
tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp (with props)
tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp (with props)
Modified:
tika/trunk/CHANGES.txt
tika/trunk/tika-parsers/pom.xml
tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Fri Apr 10 00:58:34 2015
@@ -1,3 +1,7 @@
+Release 1.9 - ???
+ * Upgrade to com.drewnoakes' metadata-extractor to 2.0 and
+ add parser for webp metadata (TIKA-1594).
+
Release 1.8 - 4/7/2015
* Duration extracted from MP3s with no ID3 tags (TIKA-1589).
Modified: tika/trunk/tika-parsers/pom.xml
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/pom.xml?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/pom.xml (original)
+++ tika/trunk/tika-parsers/pom.xml Fri Apr 10 00:58:34 2015
@@ -178,7 +178,7 @@
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
- <version>2.7.2</version>
+ <version>2.8.0</version>
</dependency>
<dependency>
<groupId>de.l3s.boilerpipe</groupId>
Modified: tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE (original)
+++ tika/trunk/tika-parsers/src/main/appended-resources/META-INF/LICENSE Fri Apr 10 00:58:34 2015
@@ -82,4 +82,13 @@ JUnRAR (https://github.com/edmund-wagner
Sqlite (bundled in org.xerial's sqlite-jdbc)
This product bundles Sqlite, which is in the Public Domain. For details
- see: https://www.sqlite.org/copyright.html
\ No newline at end of file
+ see: https://www.sqlite.org/copyright.html
+
+Two photos in test-documents (testWebp_Alpha_Lossy.webp and testWebp_Alpha_Lossless.webp)
+ are in the public domain. These files were retrieved from:
+ https://github.com/drewnoakes/metadata-extractor-images/tree/master/webp
+ These photos are also available here:
+ https://developers.google.com/speed/webp/gallery2#webp_links
+ Credits for the photo:
+ "Free Stock Photo in High Resolution - Yellow Rose 3 - Flowers"
+ Image Author: Jon Sullivan
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java Fri Apr 10 00:58:34 2015
@@ -28,19 +28,13 @@ import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.poi.util.IOUtils;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.IPTC;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.xml.sax.SAXException;
-
import com.drew.imaging.jpeg.JpegMetadataReader;
import com.drew.imaging.jpeg.JpegProcessingException;
-import com.drew.imaging.jpeg.JpegSegmentType;
+import com.drew.imaging.riff.RiffProcessingException;
import com.drew.imaging.tiff.TiffMetadataReader;
import com.drew.imaging.tiff.TiffProcessingException;
+import com.drew.imaging.webp.WebpMetadataReader;
+import com.drew.lang.ByteArrayReader;
import com.drew.lang.GeoLocation;
import com.drew.lang.Rational;
import com.drew.metadata.Directory;
@@ -55,12 +49,20 @@ import com.drew.metadata.iptc.IptcDirect
import com.drew.metadata.jpeg.JpegCommentDirectory;
import com.drew.metadata.jpeg.JpegDirectory;
import com.drew.metadata.xmp.XmpReader;
+import org.apache.poi.util.IOUtils;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.IPTC;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.xml.sax.SAXException;
/**
* Uses the <a href="http://www.drewnoakes.com/code/exif/">Metadata Extractor</a> library
* to read EXIF and IPTC image metadata and map to Tika fields.
* <p/>
* As of 2.4.0 the library supports jpeg and tiff.
+ * As of 2.8.0 the library supports webp.
*/
public class ImageMetadataExtractor {
@@ -115,6 +117,21 @@ public class ImageMetadataExtractor {
}
}
+ public void parseWebP(File file) throws IOException, TikaException {
+
+ try {
+ com.drew.metadata.Metadata webPMetadata = new com.drew.metadata.Metadata();
+ webPMetadata = WebpMetadataReader.readMetadata(file);
+ handle(webPMetadata);
+ } catch (IOException e) {
+ throw e;
+ } catch (RiffProcessingException e) {
+ throw new TikaException("Can't process Riff data", e);
+ } catch (MetadataException e) {
+ throw new TikaException("Can't process Riff data", e);
+ }
+ }
+
public void parseRawExif(InputStream stream, int length, boolean needsExifHeader)
throws IOException, SAXException, TikaException {
byte[] exif;
@@ -136,7 +153,7 @@ public class ImageMetadataExtractor {
throws IOException, SAXException, TikaException {
com.drew.metadata.Metadata metadata = new com.drew.metadata.Metadata();
ExifReader reader = new ExifReader();
- reader.extract(exifData, metadata, JpegSegmentType.APP1);
+ reader.extract(new ByteArrayReader(exifData), metadata, ExifReader.JPEG_SEGMENT_PREAMBLE.length());
try {
handle(metadata);
@@ -186,6 +203,15 @@ public class ImageMetadataExtractor {
}
}
+ private static String trimPixels(String s) {
+ //if height/width appears as "100 pixels", trim " pixels"
+ if (s != null) {
+ int i = s.lastIndexOf(" pixels");
+ s = s.substring(0, i);
+ }
+ return s;
+ }
+
/**
* Reads one or more type of Metadata Extractor fields.
*/
@@ -271,9 +297,7 @@ public class ImageMetadataExtractor {
//Exif.Image.ImageWidth Short 1 100
//Exif.Image.ImageLength Short 1 75
// and the values are found in "Thumbnail Image Width" (and Height) from Metadata Extractor
- set(directory, metadata, ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_WIDTH, Metadata.IMAGE_WIDTH);
set(directory, metadata, JpegDirectory.TAG_IMAGE_WIDTH, Metadata.IMAGE_WIDTH);
- set(directory, metadata, ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_HEIGHT, Metadata.IMAGE_LENGTH);
set(directory, metadata, JpegDirectory.TAG_IMAGE_HEIGHT, Metadata.IMAGE_LENGTH);
// Bits per sample, two methods of extracting, exif overrides jpeg
set(directory, metadata, JpegDirectory.TAG_DATA_PRECISION, Metadata.BITS_PER_SAMPLE);
@@ -426,11 +450,13 @@ public class ImageMetadataExtractor {
if (directory.containsTag(ExifIFD0Directory.TAG_RESOLUTION_UNIT)) {
metadata.set(Metadata.RESOLUTION_UNIT, directory.getDescription(ExifIFD0Directory.TAG_RESOLUTION_UNIT));
}
- if (directory.containsTag(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_WIDTH)) {
- metadata.set(Metadata.IMAGE_WIDTH, directory.getDescription(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_WIDTH));
- }
- if (directory.containsTag(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_HEIGHT)) {
- metadata.set(Metadata.IMAGE_LENGTH, directory.getDescription(ExifThumbnailDirectory.TAG_THUMBNAIL_IMAGE_HEIGHT));
+ if (directory.containsTag(ExifThumbnailDirectory.TAG_IMAGE_WIDTH)) {
+ metadata.set(Metadata.IMAGE_WIDTH,
+ trimPixels(directory.getDescription(ExifThumbnailDirectory.TAG_IMAGE_WIDTH)));
+ }
+ if (directory.containsTag(ExifThumbnailDirectory.TAG_IMAGE_HEIGHT)) {
+ metadata.set(Metadata.IMAGE_LENGTH,
+ trimPixels(directory.getDescription(ExifThumbnailDirectory.TAG_IMAGE_HEIGHT)));
}
}
Added: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java?rev=1672528&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java (added)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/image/WebPParser.java Fri Apr 10 00:58:34 2015
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+
+public class WebPParser extends AbstractParser {
+
+ /** Serial version UID */
+ private static final long serialVersionUID = -3941143576535464926L;
+
+ private static final Set<MediaType> SUPPORTED_TYPES =
+ Collections.singleton(MediaType.image("webp"));
+
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ public void parse(
+ InputStream stream, ContentHandler handler,
+ Metadata metadata, ParseContext context)
+ throws IOException, SAXException, TikaException {
+ TemporaryResources tmp = new TemporaryResources();
+ try {
+ TikaInputStream tis = TikaInputStream.get(stream, tmp);
+ new ImageMetadataExtractor(metadata).parseWebP(tis.getFile());
+ } finally {
+ tmp.dispose();
+ }
+
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+ xhtml.endDocument();
+ }
+}
Modified: tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser?rev=1672528&r1=1672527&r2=1672528&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser (original)
+++ tika/trunk/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser Fri Apr 10 00:58:34 2015
@@ -28,6 +28,7 @@ org.apache.tika.parser.image.BPGParser
org.apache.tika.parser.image.ImageParser
org.apache.tika.parser.image.PSDParser
org.apache.tika.parser.image.TiffParser
+org.apache.tika.parser.image.WebPParser
org.apache.tika.parser.iptc.IptcAnpaParser
org.apache.tika.parser.iwork.IWorkPackageParser
org.apache.tika.parser.jpeg.JpegParser
Added: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java?rev=1672528&view=auto
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java (added)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/image/WebPParserTest.java Fri Apr 10 00:58:34 2015
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.image;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.InputStream;
+
+import org.apache.tika.io.IOUtils;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.junit.Test;
+import org.xml.sax.helpers.DefaultHandler;
+
+
+public class WebPParserTest {
+
+ Parser parser = new AutoDetectParser();
+/*
+ Two photos in test-documents (testWebp_Alpha_Lossy.webp and testWebp_Alpha_Lossless.webp)
+ are in the public domain. These files were retrieved from:
+ https://github.com/drewnoakes/metadata-extractor-images/tree/master/webp
+ These photos are also available here:
+ https://developers.google.com/speed/webp/gallery2#webp_links
+ Credits for the photo:
+ "Free Stock Photo in High Resolution - Yellow Rose 3 - Flowers"
+ Image Author: Jon Sullivan
+ */
+ @Test
+ public void testSimple() throws Exception {
+ Metadata metadata = new Metadata();
+ InputStream stream =
+ getClass().getResourceAsStream("/test-documents/testWebp_Alpha_Lossy.webp");
+
+ parser.parse(stream, new DefaultHandler(), metadata, new ParseContext());
+
+ assertEquals("301", metadata.get("Image Height"));
+ assertEquals("400", metadata.get("Image Width"));
+ assertEquals("true", metadata.get("Has Alpha"));
+ assertEquals("false", metadata.get("Is Animation"));
+ assertEquals("image/webp", metadata.get(Metadata.CONTENT_TYPE));
+
+ IOUtils.closeQuietly(stream);
+
+ metadata = new Metadata();
+ stream = getClass().getResourceAsStream("/test-documents/testWebp_Alpha_Lossless.webp");
+ parser.parse(stream, new DefaultHandler(), metadata, new ParseContext());
+
+ //unfortunately, there isn't much metadata in lossless
+ assertEquals("image/webp", metadata.get(Metadata.CONTENT_TYPE));
+
+ }
+
+}
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp?rev=1672528&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossless.webp
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp?rev=1672528&view=auto
==============================================================================
Binary file - no diff available.
Propchange: tika/trunk/tika-parsers/src/test/resources/test-documents/testWebp_Alpha_Lossy.webp
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream