You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/06/30 18:31:44 UTC
[tika] branch branch_1x updated: TIKA-3459 -- integrate Drew Noakes
metadata-extractor as the underlying MP4 parser.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_1x by this push:
new f414130 TIKA-3459 -- integrate Drew Noakes metadata-extractor as the underlying MP4 parser.
f414130 is described below
commit f41413050c58befdbe4d020ba7e9d8ca4783646e
Author: tallison <ta...@apache.org>
AuthorDate: Wed Jun 30 14:30:17 2021 -0400
TIKA-3459 -- integrate Drew Noakes metadata-extractor as the underlying MP4 parser.
---
CHANGES.txt | 4 +
.../apache/tika/parser/mp4/NoakesMP4Parser.java | 302 +++++++++++++++++++++
.../apache/tika/parser/mp4/TikaMp4BoxHandler.java | 79 ++++++
.../tika/parser/mp4/boxes/TikaUserDataBox.java | 278 +++++++++++++++++++
.../services/org.apache.tika.parser.Parser | 2 +-
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 44 ++-
6 files changed, 681 insertions(+), 28 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index ac566ca..61bec23 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,9 @@
Release 1.27 - ???
+ * Migrate MP4 parsing to Drew Noakes' metadata-extractor (TIKA-3459).
+ To revert to legacy parser turn off NoakesMP4Parser and turn on MP4Parser
+ via tika-config.xml.
+
* Prevent rare infinite loop in tika-server's -spawnChild mode
when restart fails because of failure to bind to the port (TIKA-3441).
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mp4/NoakesMP4Parser.java b/tika-parsers/src/main/java/org/apache/tika/parser/mp4/NoakesMP4Parser.java
new file mode 100644
index 0000000..75d71b7
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mp4/NoakesMP4Parser.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.mp4;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import com.drew.imaging.mp4.Mp4Reader;
+import com.drew.metadata.Directory;
+import com.drew.metadata.MetadataException;
+import com.drew.metadata.mp4.Mp4BoxHandler;
+import com.drew.metadata.mp4.Mp4Directory;
+import com.drew.metadata.mp4.media.Mp4SoundDirectory;
+import com.drew.metadata.mp4.media.Mp4VideoDirectory;
+import org.apache.commons.lang3.StringUtils;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.exception.RuntimeSAXException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.XMPDM;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AbstractParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+/**
+ * Parser for the MP4 media container format, as well as the older
+ * QuickTime format that MP4 is based on.
+ * <p>
+ * This uses Drew Noakes' metadata-extractor: https://github.com/drewnoakes/metadata-extractor
+ */
+public class NoakesMP4Parser extends AbstractParser {
+ /**
+ * Serial version UID
+ */
+ private static final long serialVersionUID = 84011216792285L;
+ private static final Map<MediaType, List<String>> typesMap = new HashMap<>();
+ private static final Set<MediaType> SUPPORTED_TYPES =
+ Collections.unmodifiableSet(typesMap.keySet());
+
+ private static final MediaType APPLICATION_MP4 = MediaType.application("mp4");
+ private static final int MAX_ERROR_MESSAGES = 100;
+ static {
+ // All types should be 4 bytes long, space padded as needed
+ typesMap.put(MediaType.audio("mp4"), Arrays.asList("M4A ", "M4B ", "F4A ", "F4B "));
+ typesMap.put(MediaType.video("3gpp"),
+ Arrays.asList("3ge6", "3ge7", "3gg6", "3gp1", "3gp2", "3gp3", "3gp4", "3gp5",
+ "3gp6", "3gs7"));
+ typesMap.put(MediaType.video("3gpp2"), Arrays.asList("3g2a", "3g2b", "3g2c"));
+ typesMap.put(MediaType.video("mp4"), Arrays.asList("mp41", "mp42"));
+ typesMap.put(MediaType.video("x-m4v"), Arrays.asList("M4V ", "M4VH", "M4VP"));
+
+ typesMap.put(MediaType.video("quicktime"), Collections.emptyList());
+ typesMap.put(MediaType.application("mp4"), Collections.emptyList());
+ }
+
+ private ISO6709Extractor iso6709Extractor = new ISO6709Extractor();
+
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
+ ParseContext context) throws IOException, SAXException, TikaException {
+
+ TemporaryResources tmp = new TemporaryResources();
+ TikaInputStream tstream = TikaInputStream.get(stream, tmp);
+
+ try (InputStream is = Files.newInputStream(tstream.getPath())) {
+
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+ com.drew.metadata.Metadata mp4Metadata = new com.drew.metadata.Metadata();
+ Mp4BoxHandler boxHandler = new TikaMp4BoxHandler(mp4Metadata, metadata, xhtml);
+ try {
+ Mp4Reader.extract(is, boxHandler);
+ } catch (RuntimeSAXException e) {
+ throw (SAXException) e.getCause();
+ }
+ //TODO -- figure out how to get IOExceptions out of boxhandler. Mp4Reader
+ //currently swallows IOExceptions.
+ Set<String> errorMessages =
+ processMp4Directories(
+ mp4Metadata.getDirectoriesOfType(Mp4Directory.class),
+ metadata);
+
+ for (String m : errorMessages) {
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, m);
+ }
+ xhtml.endDocument();
+ } finally {
+ tmp.dispose();
+ }
+ }
+
+ private Set<String> processMp4Directories(Collection<Mp4Directory> mp4Directories,
+ Metadata metadata) {
+ Set<String> errorMsgs = new HashSet<>();
+ for (Mp4Directory mp4Directory : mp4Directories) {
+ for (String m : mp4Directory.getErrors()) {
+ if (errorMsgs.size() < MAX_ERROR_MESSAGES) {
+ errorMsgs.add(m);
+ } else {
+ break;
+ }
+ }
+/* for (Tag t : mp4Directory.getTags()) {
+ System.out.println(mp4Directory.getClass() + " : " + t.getTagName()
+ + " : " + mp4Directory.getString(t.getTagType()));
+ }*/
+ if (mp4Directory instanceof Mp4SoundDirectory) {
+ processMp4SoundDirectory((Mp4SoundDirectory) mp4Directory, metadata);
+ } else if (mp4Directory instanceof Mp4VideoDirectory) {
+ processMp4VideoDirectory((Mp4VideoDirectory) mp4Directory, metadata);
+ } else {
+ processActualMp4Directory(mp4Directory, metadata);
+ }
+ }
+ return errorMsgs;
+ }
+
+ private void processMp4VideoDirectory(Mp4VideoDirectory mp4Directory, Metadata metadata) {
+ addInt(mp4Directory, metadata, Mp4VideoDirectory.TAG_HEIGHT, Metadata.IMAGE_LENGTH);
+ addInt(mp4Directory, metadata, Mp4VideoDirectory.TAG_WIDTH, Metadata.IMAGE_WIDTH);
+ if (mp4Directory.containsTag(Mp4VideoDirectory.TAG_COMPRESSOR_NAME)) {
+ String compressor = mp4Directory.getString(Mp4VideoDirectory.TAG_COMPRESSOR_NAME);
+ metadata.set(XMPDM.VIDEO_COMPRESSOR, compressor);
+ }
+ }
+
+ private void processMp4SoundDirectory(Mp4SoundDirectory mp4SoundDirectory,
+ Metadata metadata) {
+ addInt(mp4SoundDirectory, metadata, Mp4SoundDirectory.TAG_AUDIO_SAMPLE_RATE,
+ XMPDM.AUDIO_SAMPLE_RATE);
+
+ try {
+ int numChannels = mp4SoundDirectory.getInt(Mp4SoundDirectory.TAG_NUMBER_OF_CHANNELS);
+
+ if (numChannels == 1) {
+ metadata.set(XMPDM.AUDIO_CHANNEL_TYPE, "Mono");
+ } else if (numChannels == 2) {
+ metadata.set(XMPDM.AUDIO_CHANNEL_TYPE, "Stereo");
+ } else {
+ //??? log
+ }
+
+ } catch (MetadataException e) {
+ //log
+ }
+ }
+
+ private void addInt(Mp4Directory mp4Directory, Metadata metadata, int tag,
+ Property property) {
+ try {
+ int val = mp4Directory.getInt(tag);
+ metadata.set(property, val);
+ } catch (MetadataException e) {
+ //log
+ }
+ }
+
+ private void processActualMp4Directory(Mp4Directory mp4Directory, Metadata metadata) {
+ addDate(mp4Directory, metadata, Mp4Directory.TAG_CREATION_TIME, TikaCoreProperties.CREATED);
+ addDate(mp4Directory, metadata, Mp4Directory.TAG_MODIFICATION_TIME,
+ TikaCoreProperties.MODIFIED);
+ handleBrands(mp4Directory, metadata);
+ handleDurationInSeconds(mp4Directory, metadata);
+
+ addDouble(mp4Directory, metadata, Mp4Directory.TAG_LATITUDE, TikaCoreProperties.LATITUDE);
+ addDouble(mp4Directory, metadata, Mp4Directory.TAG_LONGITUDE, TikaCoreProperties.LONGITUDE);
+ addInt(mp4Directory, metadata, Mp4Directory.TAG_TIME_SCALE, XMPDM.AUDIO_SAMPLE_RATE);
+ }
+
+ private void handleDurationInSeconds(Mp4Directory mp4Directory, Metadata metadata) {
+ String durationInSeconds = mp4Directory.getString(Mp4Directory.TAG_DURATION_SECONDS);
+ if (durationInSeconds == null) {
+ return;
+ }
+ if (! durationInSeconds.contains("/")) {
+ try {
+ double d = Double.parseDouble(durationInSeconds);
+ DecimalFormat df =
+ (DecimalFormat) NumberFormat.getNumberInstance(Locale.ROOT);
+ df.applyPattern("0.0#");
+ metadata.set(XMPDM.DURATION, df.format(d));
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+ return;
+ }
+ String[] bits = durationInSeconds.split("/");
+ if (bits.length != 2) {
+ return;
+ }
+ double durationSeconds;
+ try {
+ long numerator = Long.parseLong(bits[0]);
+ long denominator = Long.parseLong(bits[1]);
+ if (denominator != 0) {
+ durationSeconds = (double) numerator / (double) denominator;
+ // Get the duration
+ //TODO Replace this with a 2dp Duration Property Converter
+ //avoid thread safety issues by creating a new decimal format for every call
+ //threadlocal doesn't play well in long running processes.
+ DecimalFormat df =
+ (DecimalFormat) NumberFormat.getNumberInstance(Locale.ROOT);
+ df.applyPattern("0.0#");
+ metadata.set(XMPDM.DURATION, df.format(durationSeconds));
+ }
+ } catch (NumberFormatException e) {
+ //log
+ return;
+ }
+ }
+
+ private void handleBrands(Mp4Directory mp4Directory, Metadata metadata) {
+
+
+ String majorBrand = mp4Directory.getString(Mp4Directory.TAG_MAJOR_BRAND);
+ // Identify the type based on the major brand
+ Optional<MediaType> typeHolder = typesMap.entrySet().stream()
+ .filter(e -> e.getValue().contains(majorBrand)).findFirst()
+ .map(Map.Entry::getKey);
+
+ if (!typeHolder.isPresent()) {
+ String compatibleBrands =
+ mp4Directory.getString(Mp4Directory.TAG_COMPATIBLE_BRANDS);
+ if (compatibleBrands != null) {
+ // If no match for major brand, see if any of the compatible brands match
+ typeHolder = typesMap.entrySet().stream().filter(e ->
+ e.getValue().stream().anyMatch(compatibleBrands::contains))
+ .findFirst().map(Map.Entry::getKey);
+ }
+ }
+ MediaType type = typeHolder.orElse(MediaType.application("mp4"));
+ if (metadata.getValues(Metadata.CONTENT_TYPE) == null) {
+ metadata.set(Metadata.CONTENT_TYPE, type.toString());
+ } else if (! type.equals(APPLICATION_MP4)) { //todo check for specialization?
+ metadata.set(Metadata.CONTENT_TYPE, type.toString());
+ }
+ if (type.getType().equals("audio") && ! StringUtils.isBlank(majorBrand)) {
+ metadata.set(XMPDM.AUDIO_COMPRESSOR, majorBrand.trim());
+ }
+
+ }
+
+ private void addDate(Mp4Directory mp4Directory, Metadata metadata, int tag,
+ Property property) {
+ Date d = mp4Directory.getDate(tag);
+ if (d == null) {
+ return;
+ }
+ metadata.set(property, d);
+
+ }
+
+ private void addDouble(Directory mp4Directory, Metadata metadata, int tag,
+ Property property) {
+ try {
+ double val = mp4Directory.getDouble(tag);
+ metadata.set(property, val);
+ } catch (MetadataException e) {
+ //log
+ return;
+ }
+
+ }
+}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java
new file mode 100644
index 0000000..0675110
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.mp4;
+
+import java.io.IOException;
+
+import com.drew.imaging.mp4.Mp4Handler;
+import com.drew.lang.annotations.NotNull;
+import com.drew.lang.annotations.Nullable;
+import com.drew.metadata.Metadata;
+import com.drew.metadata.mp4.Mp4BoxHandler;
+import com.drew.metadata.mp4.Mp4Context;
+import com.drew.metadata.mp4.boxes.Box;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.parser.mp4.boxes.TikaUserDataBox;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+public class TikaMp4BoxHandler extends Mp4BoxHandler {
+
+ org.apache.tika.metadata.Metadata tikaMetadata;
+ final XHTMLContentHandler xhtml;
+ public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata tikaMetadata,
+ XHTMLContentHandler xhtml) {
+ super(metadata);
+ this.tikaMetadata = tikaMetadata;
+ this.xhtml = xhtml;
+ }
+
+ @Override
+ public boolean shouldAcceptBox(@NotNull Box box) {
+ if (box.type.equals("udta")) {
+ return true;
+ }
+ return super.shouldAcceptBox(box);
+ }
+
+ @Override
+ public boolean shouldAcceptContainer(@NotNull Box box) {
+ return super.shouldAcceptContainer(box);
+ }
+
+ @Override
+ public Mp4Handler<?> processBox(@NotNull Box box, @Nullable byte[] payload, Mp4Context context)
+ throws IOException {
+ if (box.type.equals("udta")) {
+ return processUserData(box, payload, context);
+ }
+
+ return super.processBox(box, payload, context);
+ }
+
+
+ private Mp4Handler<?> processUserData(Box box, byte[] payload, Mp4Context context) throws IOException {
+ if (payload == null) {
+ return this;
+ }
+ try {
+ new TikaUserDataBox(box, payload, tikaMetadata, xhtml).addMetadata(directory);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ }
+ return this;
+ }
+}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java b/tika-parsers/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java
new file mode 100644
index 0000000..2e30ff3
--- /dev/null
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.mp4.boxes;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.drew.lang.SequentialByteArrayReader;
+import com.drew.lang.SequentialReader;
+import com.drew.lang.annotations.NotNull;
+import com.drew.lang.annotations.Nullable;
+import com.drew.metadata.mp4.Mp4Directory;
+import com.drew.metadata.mp4.boxes.Box;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.exception.RuntimeSAXException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.XMP;
+import org.apache.tika.metadata.XMPDM;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+public class TikaUserDataBox extends Box {
+
+ private static final String LOCATION_CODE = "\u00A9xyz";
+ private static final String META = "meta";
+ private static final String ILST = "ilst";
+ private static final String MDTA = "mdta";
+ private static final String HDLR = "hdlr";
+ private static final String MDIR = "mdir";//apple metadata itunes reader
+ private static final Pattern COORDINATE_PATTERN =
+ Pattern.compile("([+-]\\d+\\.\\d+)([+-]\\d+\\.\\d+)");
+
+ @Nullable
+ private String coordinateString;
+
+ private boolean isQuickTime = false;
+ private final Metadata metadata;
+ private final XHTMLContentHandler xhtml;
+ public TikaUserDataBox(@NotNull Box box, byte[] payload, Metadata metadata,
+ XHTMLContentHandler xhtml) throws IOException, SAXException {
+ super(box);
+ this.metadata = metadata;
+ this.xhtml = xhtml;
+ int length = payload.length;
+ SequentialReader reader = new SequentialByteArrayReader(payload);
+ while (reader.getPosition() < (long) length) {
+ long size = reader.getUInt32();
+ if (size <= 4L) {
+ break;
+ }
+ String kindName = reader.getString(4, StandardCharsets.ISO_8859_1);
+ if (LOCATION_CODE.equals(kindName)) {
+ int xyzLength = reader.getUInt16();
+ reader.skip(2L);
+ this.coordinateString = reader.getString(xyzLength, "UTF-8");
+ } else if (META.equals(kindName)) {
+ reader.getUInt32();//not sure what this is
+ long lengthToStartOfList = reader.getUInt32() - 4;//this is the length to
+ // 'ilst', but the length of the ilist is defined in the 4 bytes before ilist
+ if (lengthToStartOfList < 0 || lengthToStartOfList > Integer.MAX_VALUE) {
+ return;
+ }
+ String hdlr = reader.getString(4, StandardCharsets.ISO_8859_1);
+ reader.getUInt32();
+ reader.getUInt32();
+ String subtype = reader.getString(4, StandardCharsets.ISO_8859_1);
+ // If the second and the fifth 32-bit integers encode 'hdlr' and 'mdta' respectively
+ // then the MetaBox is formatted according to QuickTime File Format.
+ // See https://developer.apple.com/library/content/documentation
+ // /QuickTime/QTFF/Metadata/Metadata.html
+ if (HDLR.equals(hdlr) && MDTA.equals(subtype)) {
+ isQuickTime = true;
+ }
+ int read = 16;//bytes read so far
+ parseUserDataBox(reader, subtype, read, (int)lengthToStartOfList);
+ } else {
+ if (size < 8L) {
+ return;
+ }
+
+ reader.skip(size - 8L);
+ }
+ }
+
+ }
+
+ private void parseUserDataBox(SequentialReader reader, String handlerType,
+ int read, int lengthToStartOfList)
+ throws IOException {
+ if (!MDIR.equals(handlerType)) {
+ return;
+ }
+ if (lengthToStartOfList < read) {
+ return;
+ }
+ int toSkip = lengthToStartOfList - read;
+ reader.skip(toSkip);
+ long len = reader.getUInt32();
+ if (len >= Integer.MAX_VALUE || len <= 0) {
+ //log
+ return;
+ }
+ String subType = reader.getString(4, StandardCharsets.ISO_8859_1);
+ //this handles "free" types...not sure if there are others?
+ //will throw IOException if no ilist is found
+ while (! subType.equals(ILST)) {
+ reader.skip(len - 8);
+ len = reader.getUInt32();
+ subType = reader.getString(4, StandardCharsets.ISO_8859_1);
+ }
+ if (ILST.equals(subType)) {
+ processIList(reader, len);
+ }
+
+ }
+
+
+
+ private void processIList(SequentialReader reader, long totalLen)
+ throws IOException {
+
+ long totalRead = 0;
+ while (totalRead < totalLen) {
+ long recordLen = reader.getUInt32();
+ String fieldName = reader.getString(4, StandardCharsets.ISO_8859_1);
+ long fieldLen = reader.getUInt32();
+ String typeName = reader.getString(4, StandardCharsets.ISO_8859_1);//data
+ totalRead += 16;
+ if ("data".equals(typeName)) {
+ reader.skip(8);//not sure what these are
+ totalRead += 8;
+ int toRead = (int) fieldLen - 16;
+ if (toRead <= 0) {
+ //log?
+ return;
+ }
+ if ("covr".equals(fieldName)) {
+ //covr can be an image file, e.g. png or jpeg
+ //skip this for now
+ reader.skip(toRead);
+ } else if ("cpil".equals(fieldName)) {
+ int compilationId = (int)reader.getByte();
+ metadata.set(XMPDM.COMPILATION, compilationId);
+ } else if ("trkn".equals(fieldName)) {
+ if (toRead == 8) {
+ long numA = reader.getUInt32();
+ long numB = reader.getUInt32();
+ metadata.set(XMPDM.TRACK_NUMBER, (int)numA);
+ } else {
+ //log
+ reader.skip(toRead);
+ }
+ } else if ("disk".equals(fieldName)) {
+ int a = reader.getInt32();
+ short b = reader.getInt16();
+ metadata.set(XMPDM.DISC_NUMBER, a);
+ } else {
+ String val = reader.getString(toRead, StandardCharsets.UTF_8);
+ try {
+ addMetadata(fieldName, val);
+ } catch (SAXException e) {
+ //need to punch through IOException catching in MP4Reader
+ throw new RuntimeSAXException(e);
+ }
+ }
+
+ totalRead += toRead;
+ } else {
+ int toSkip = (int) recordLen - 16;
+ if (toSkip <= 0) {
+ //log?
+ return;
+ }
+ reader.skip(toSkip);
+ totalRead += toSkip;
+ }
+ }
+ }
+
+
+ private void addMetadata(String key, String value) throws SAXException {
+ switch (key) {
+ case "\u00A9nam":
+ metadata.set(TikaCoreProperties.TITLE, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9too":
+ metadata.set(XMP.CREATOR_TOOL, value);
+ break;
+ case "\u00A9ART" :
+ metadata.set(XMPDM.ARTIST, value);
+ metadata.set(TikaCoreProperties.CREATOR, value);
+ xhtml.element("p", value);
+ break;
+ case "aART" :
+ metadata.set(XMPDM.ALBUM_ARTIST, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9wrt":
+ metadata.set(XMPDM.COMPOSER, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9alb":
+ metadata.set(XMPDM.ALBUM, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9gen" :
+ metadata.set(XMPDM.GENRE, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9day" :
+ //this can be a year "2008" or a date "2017-04-26T07:00:00Z"
+ metadata.set(XMPDM.RELEASE_DATE, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9cmt" :
+ metadata.set(XMPDM.LOG_COMMENT, value);
+ xhtml.element("p", value);
+ break;
+ case "cprt" :
+ metadata.set(XMPDM.COPYRIGHT, value);
+ xhtml.element("p", value);
+ break;
+ case "keyw" :
+ metadata.set(Metadata.SUBJECT, value);
+ xhtml.element("p", value);
+ break;
+ case "\u00A9lyr" :
+ xhtml.element("p", value);
+ break;
+ case "ldes" :
+ metadata.set(TikaCoreProperties.DESCRIPTION, value);
+ xhtml.element("p", value);
+ case "xid " :
+ //not sure this is the right use of this key
+ metadata.set(XMP.IDENTIFIER, value);
+ break;
+ //purd date?
+ //xid ? e.g. SonyBMG:isrc:KRA031208874
+ //cprt copyright
+ //ownr ? and apID
+ //flvr ?
+ //son = nam, soal = (c)alb soar = aART?
+ //(C)ART
+ }
+ }
+
+ public void addMetadata(Mp4Directory directory) {
+ if (this.coordinateString != null) {
+ Matcher matcher = COORDINATE_PATTERN.matcher(this.coordinateString);
+ if (matcher.find()) {
+ double latitude = Double.parseDouble(matcher.group(1));
+ double longitude = Double.parseDouble(matcher.group(2));
+ directory.setDouble(8193, latitude);
+ directory.setDouble(8194, longitude);
+ }
+ }
+ }
+}
+
+
diff --git a/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser b/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
index 45799e6..69c250e 100644
--- a/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
+++ b/tika-parsers/src/main/resources/META-INF/services/org.apache.tika.parser.Parser
@@ -55,7 +55,7 @@ org.apache.tika.parser.microsoft.xml.WordMLParser
org.apache.tika.parser.microsoft.xml.SpreadsheetMLParser
org.apache.tika.parser.mif.MIFParser
org.apache.tika.parser.mp3.Mp3Parser
-org.apache.tika.parser.mp4.MP4Parser
+org.apache.tika.parser.mp4.NoakesMP4Parser
org.apache.tika.parser.hdf.HDFParser
org.apache.tika.parser.netcdf.NetCDFParser
org.apache.tika.parser.odf.FlatOpenDocumentParser
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
index 52b192b..369cf88 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
@@ -18,12 +18,10 @@ package org.apache.tika.parser.mp4;
import static org.junit.Assert.assertEquals;
-import java.io.EOFException;
-import java.io.InputStream;
-import java.nio.file.Paths;
+import org.junit.Test;
+import org.xml.sax.ContentHandler;
import org.apache.tika.TikaTest;
-import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -31,9 +29,6 @@ import org.apache.tika.metadata.XMP;
import org.apache.tika.metadata.XMPDM;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.ContentHandlerFactory;
-import org.junit.Test;
-import org.xml.sax.ContentHandler;
/**
* Test case for parsing mp4 files.
@@ -41,7 +36,7 @@ import org.xml.sax.ContentHandler;
public class MP4ParserTest extends TikaTest {
/**
* Test that we can extract information from
- * a M4A MP4 Audio file
+ * a M4A MP4 Audio file
*/
@Test
public void testMP4ParsingAudio() throws Exception {
@@ -65,7 +60,7 @@ public class MP4ParserTest extends TikaTest {
assertContains("2008", content);
assertContains("Test Comment", content);
assertContains("Test Genre", content);
-
+
// Check XMPDM-typed audio properties
assertEquals("Test Album", metadata.get(XMPDM.ALBUM));
assertEquals("Test Artist", metadata.get(XMPDM.ARTIST));
@@ -77,42 +72,37 @@ public class MP4ParserTest extends TikaTest {
assertEquals("Test Album Artist", metadata.get(XMPDM.ALBUM_ARTIST));
assertEquals("6", metadata.get(XMPDM.DISC_NUMBER));
assertEquals("0", metadata.get(XMPDM.COMPILATION));
-
-
+
+
assertEquals("44100", metadata.get(XMPDM.AUDIO_SAMPLE_RATE));
assertEquals("Stereo", metadata.get(XMPDM.AUDIO_CHANNEL_TYPE));
assertEquals("M4A", metadata.get(XMPDM.AUDIO_COMPRESSOR));
assertEquals("0.07", metadata.get(XMPDM.DURATION));
-
+
assertEquals("iTunes 10.5.3.3", metadata.get(XMP.CREATOR_TOOL));
-
-
+
+
// Check again by file, rather than stream
- TikaInputStream tstream = TikaInputStream.get(
- MP4ParserTest.class.getResourceAsStream("/test-documents/testMP4.m4a"));
+ TikaInputStream tstream = TikaInputStream
+ .get(MP4ParserTest.class.getResourceAsStream("/test-documents/testMP4.m4a"));
tstream.getFile();
ContentHandler handler = new BodyContentHandler();
try {
- AUTO_DETECT_PARSER.parse(tstream, handler, metadata, new ParseContext());
+ AUTO_DETECT_PARSER.parse(tstream, handler, metadata, new ParseContext());
} finally {
- tstream.close();
+ tstream.close();
}
//TODO: why don't we check the output here?
}
-
+
// TODO Test a MP4 Video file
// TODO Test an old QuickTime Video File
@Test(timeout = 30000)
public void testInfiniteLoop() throws Exception {
//test that a truncated mp4 doesn't cause an infinite loop
//TIKA-1931 and TIKA-1924
- try {
- XMLResult r = getXML("testMP4_truncated.m4a");
- assertEquals("audio/mp4", r.metadata.get(Metadata.CONTENT_TYPE));
- assertEquals("M4A", r.metadata.get(XMPDM.AUDIO_COMPRESSOR));
- } catch (TikaException e) {
- //happens with Java 8
- //should be eof
- }
+ XMLResult r = getXML("testMP4_truncated.m4a");
+ assertEquals("audio/mp4", r.metadata.get(Metadata.CONTENT_TYPE));
+ assertEquals("M4A", r.metadata.get(XMPDM.AUDIO_COMPRESSOR));
}
}