You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2026/09/30 15:51:16 UTC
(tika) branch main updated: TIKA-4917: improve date parsing (#3225)
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new d9c9cd1ac5 TIKA-4917: improve date parsing (#3225)
d9c9cd1ac5 is described below
commit d9c9cd1ac53663bc931d5048939e3fb3968bf3f8
Author: Tim Allison <ta...@apache.org>
AuthorDate: Wed Sep 30 11:51:10 2026 -0400
TIKA-4917: improve date parsing (#3225)
---
CHANGES.txt | 10 +
docs/modules/ROOT/nav.adoc | 1 +
.../modules/ROOT/pages/advanced/date-handling.adoc | 65 +++
docs/modules/ROOT/pages/advanced/index.adoc | 1 +
.../ROOT/pages/configuration/metadata-filters.adoc | 8 +-
.../java/org/apache/tika/metadata/Metadata.java | 24 +-
.../filter/DateNormalizingMetadataFilter.java | 78 +--
.../main/java/org/apache/tika/utils/DateUtils.java | 70 +--
.../main/java/org/apache/tika/utils/TikaDates.java | 527 +++++++++++++++++++++
.../tika/metadata/MetadataZonelessDateTest.java | 72 +++
.../java/org/apache/tika/utils/TikaDatesTest.java | 408 ++++++++++++++++
tika-parent/pom.xml | 2 +-
.../tika-parsers-standard-modules/pom.xml | 2 +-
.../java/org/apache/tika/parser/mp3/Mp3Parser.java | 11 +-
.../tika/parser/mp4/boxes/TikaUserDataBox.java | 6 +-
.../org/apache/tika/parser/ogg/OggAudioParser.java | 6 +-
.../apache/tika/parser/font/TrueTypeParser.java | 13 +-
.../tika/parser/image/ImageMetadataExtractor.java | 19 +-
.../parser/image/ImageMetadataExtractorTest.java | 26 +-
.../apache/tika/parser/image/JpegParserTest.java | 5 +-
.../tika-parser-mail-commons/pom.xml | 13 +-
.../tika/parser/mailcommons/MailDateParser.java | 3 +
.../apache/tika/parser/mailcommons/MailUtil.java | 100 +---
.../tika-parser-mail-module/pom.xml | 11 +-
.../tika/parser/mail/MailContentHandler.java | 25 +-
.../org/apache/tika/parser/mail}/MailUtil.java | 2 +-
.../org/apache/tika/parser/mbox/MboxParser.java | 18 +-
.../org/apache/tika/parser/mail}/MailUtilTest.java | 2 +-
.../apache/tika/parser/mail/RFC822ParserTest.java | 53 ++-
.../tika-parser-microsoft-module/pom.xml | 13 +-
.../tika/parser/microsoft/OutlookExtractor.java | 14 +-
.../tika/parser/microsoft/SummaryExtractor.java | 4 +-
.../ooxml/OOXMLWordAndPowerPointTextHandler.java | 6 +-
.../tika/parser/microsoft/rtf/TextExtractor.java | 37 +-
.../tika/parser/microsoft/rtf/RTFParserTest.java | 28 +-
.../tika/parser/odf/DateElementHandler.java} | 72 +--
.../tika/parser/odf/OpenDocumentMetaParser.java | 5 +-
.../org/apache/tika/parser/odf/ODFParserTest.java | 35 ++
.../tika/parser/pkg/AbstractArchiveParser.java | 49 +-
.../apache/tika/parser/pkg/CompressorParser.java | 11 +-
.../org/apache/tika/parser/pkg/PackageParser.java | 9 +-
.../java/org/apache/tika/parser/pkg/RarParser.java | 8 +-
.../org/apache/tika/parser/pkg/SevenZParser.java | 4 +
.../java/org/apache/tika/parser/pkg/ZipParser.java | 110 ++++-
.../apache/tika/parser/pkg/AbstractPkgTest.java | 9 +-
.../tika/parser/pkg/ArchiveTimeBoundsTest.java | 75 +++
.../org/apache/tika/parser/pkg/GzipParserTest.java | 29 ++
.../apache/tika/parser/pkg/ZipDosTimeDstTest.java | 101 ++++
.../apache/tika/parser/pkg/ZipEntryTimesTest.java | 101 ++++
.../apache/tika/parser/xml/MetadataHandler.java | 4 +-
.../tika-parser-xmp-commons/pom.xml | 11 -
.../java/org/apache/tika/parser/xmp/XmpDates.java | 36 +-
.../org/apache/tika/parser/xmp/XmpExtractor.java | 5 +-
.../org/apache/tika/parser/xmp/XmpDatesTest.java | 33 +-
.../apache/tika/parser/xmp/XmpExtractorTest.java | 23 +
.../tika/pipes/emitter/jdbc/JDBCEmitter.java | 39 +-
.../tika/pipes/emitter/jdbc/JDBCEmitterTest.java | 39 ++
.../tika/metadata/filter/TestMetadataFilter.java | 53 +++
58 files changed, 2082 insertions(+), 462 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 9d213462a5..687d099986 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,15 @@
Release 4.2.0 - unreleased
+ * Refactor date parsing for more consistent, accurate dates. File-system
+ times from archive entries and mail attachments move to fs:created/
+ fs:modified. MailDateParser is superseded by TikaDates and MailUtil
+ moved to org.apache.tika.parser.mail in tika-parser-mail-module;
+ tika-parser-mail-commons remains as a deprecated shim and will be
+ removed in 4.3.0. DateNormalizingMetadataFilter honors explicit
+ offsets, normalizes every value of a multi-valued key, and fills in
+ partial dates (2019 -> 2019-01-01T12:00:00Z) so date fields never
+ reject a document (TIKA-4917).
+
* XmlSecurityContractTest pins XMLReaderUtils' XXE and entity-expansion defenses
forbidden-apis now rejects building JAXP or Commons Secure XML parsers anywhere but
XMLReaderUtils (TIKA-4939).
diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
index 6ee158b028..10d3a30966 100644
--- a/docs/modules/ROOT/nav.adoc
+++ b/docs/modules/ROOT/nav.adoc
@@ -90,6 +90,7 @@
** xref:advanced/charset-detection-design.adoc[Charset Detection Pipeline]
** xref:advanced/zip-detection.adoc[ZIP Detection and Salvaging]
** xref:advanced/embedded-documents.adoc[Embedded Document Metadata]
+** xref:advanced/date-handling.adoc[Date Handling]
** xref:advanced/local-vlm-server.adoc[Running a Local VLM Server]
** Integration Testing
*** xref:advanced/integration-testing/tika-app.adoc[Testing with Tika App]
diff --git a/docs/modules/ROOT/pages/advanced/date-handling.adoc b/docs/modules/ROOT/pages/advanced/date-handling.adoc
new file mode 100644
index 0000000000..62ff01fd0f
--- /dev/null
+++ b/docs/modules/ROOT/pages/advanced/date-handling.adoc
@@ -0,0 +1,65 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+= Date Handling
+
+Since TIKA-4917 in 4.2.0, one parser in tika-core, `org.apache.tika.utils.TikaDates`, parses
+every date string a document supplies. It replaced three parsers with different rules
+(`DateUtils`, `MailDateParser`, and the XMP path), which had stamped some zone-less values with
+`Z`, resolved others in the server's time zone, and inflated partial or invalid dates into
+full timestamps. Dates now depend only on what the document says.
+
+== Storage forms
+
+A `DATE`-typed metadata value takes one of three shapes:
+
+[cols="1,2,2"]
+|===
+|Document says |Stored as |Example
+
+|Zone or offset |Canonical UTC |`2024-03-05T14:07:00Z`
+|Wall clock, no zone |Zone-less |`2024-03-05T14:07:00`
+|Date only |Date only |`2024-03-05`
+|===
+
+Partial dates (`2019`, `2019-06`) and values Tika cannot parse stay on the format's own key
+(for example `xmpDM:releaseDate`) and are never promoted to `dcterms:created` or
+`dcterms:modified`. Years outside 1000..2200 are rejected.
+
+== Reading dates back
+
+`Metadata.getDate()` reads a zone-less value as UTC, never as the JVM default zone, and a
+date-only value as midday UTC. Partial and out-of-range values return `null`.
+
+Endpoints that require a full timestamp (Solr `pdate`, for example) still need the
+xref:configuration/metadata-filters.adoc[`date-normalizing-metadata-filter`]. It resolves
+zone-less and date-only values in a configured default zone, and it *invents* the missing
+fields of a partial date: `2019` becomes January 1, `2019-06` becomes June 1, both at midday.
+Tika itself never does this; only the filter does, and only because a rejected date field
+would otherwise reject the whole document.
+
+== File-system times
+
+Entry times from archives (zip, 7z, tar, rar, arj, cpio) and mail attachments are file-system
+facts, not document facts. They are on `fs:created`, `fs:modified`, and `fs:accessed`, not on
+`dcterms:*`. DOS-format times are stored zone-less.
+
+== API
+
+`TikaDates` is the supported parser. `MailDateParser` and `org.apache.tika.parser.mailcommons.MailUtil`
+remain in `tika-parser-mail-commons` as a deprecated shim through 4.2.x and are removed in
+4.3.0; `MailUtil` lives in `org.apache.tika.parser.mail` in `tika-parser-mail-module`.
diff --git a/docs/modules/ROOT/pages/advanced/index.adoc b/docs/modules/ROOT/pages/advanced/index.adoc
index 2f1045a417..ab15bb4a9d 100644
--- a/docs/modules/ROOT/pages/advanced/index.adoc
+++ b/docs/modules/ROOT/pages/advanced/index.adoc
@@ -49,6 +49,7 @@ prioritize the gap.
== Parsing internals
* xref:advanced/embedded-documents.adoc[Embedded Document Metadata] — how Tika tracks embedded documents and their paths
+* xref:advanced/date-handling.adoc[Date Handling] — how dates are parsed, stored, and read back since 4.2.0
* xref:advanced/local-vlm-server.adoc[Running a Local VLM Server] — an OpenAI-compatible endpoint for air-gapped OCR
== Integration testing
diff --git a/docs/modules/ROOT/pages/configuration/metadata-filters.adoc b/docs/modules/ROOT/pages/configuration/metadata-filters.adoc
index e1770b4c47..b5c4ebe1cc 100644
--- a/docs/modules/ROOT/pages/configuration/metadata-filters.adoc
+++ b/docs/modules/ROOT/pages/configuration/metadata-filters.adoc
@@ -81,8 +81,12 @@ matches. Use it to keep, say, embedded images out of the output entirely.
`RENDERING`, `VERSION`, `ALTERNATE_FORMAT_CHUNK` (empty). An unrecognized name is a config error.
|`date-normalizing-metadata-filter`
-|Rewrites every `DATE`-typed field that has no timezone to `yyyy-MM-dd'T'HH:mm:ss'Z'`, assuming
-the configured zone. Values that already end in `Z` are left alone.
+|Rewrites every `DATE`-typed value to `yyyy-MM-dd'T'HH:mm:ss'Z'` so that date fields in
+Solr, Elasticsearch, and OpenSearch never reject a document. A zone-less value is resolved in
+the configured zone, a date-only value as midday there, and an explicit offset is converted.
+*Partial dates are invented, not dropped*: `2019` becomes `2019-01-01T12:00:00Z` and
+`2019-06` becomes `2019-06-01T12:00:00Z` (in the configured zone). Unparseable values pass
+through with a warning. See xref:advanced/date-handling.adoc[Date Handling].
|`defaultTimeZone` — zone id, e.g. `America/New_York` (`UTC`)
|`capture-group-metadata-filter`
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
index f995e42fea..91e729feaa 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java
@@ -31,6 +31,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.TimeZone;
import org.slf4j.Logger;
@@ -40,8 +41,8 @@ import org.apache.tika.metadata.Property.PropertyType;
import org.apache.tika.metadata.writelimiter.MetadataWriteLimiter;
import org.apache.tika.metadata.writelimiter.MetadataWriteLimiterFactory;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.utils.DateUtils;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
* A multi-valued metadata container.
@@ -113,7 +114,6 @@ public class Metadata implements Serializable {
* Some parsers will have the date as a ISO-8601 string
* already, and will set that into the Metadata object.
*/
- private static final DateUtils DATE_UTILS = new DateUtils();
/**
* A map of all metadata attributes.
*/
@@ -174,16 +174,10 @@ public class Metadata implements Serializable {
return sdf;
}
- /**
- * Parses the given date string. This method is synchronized to prevent
- * concurrent access to the thread-unsafe date formats.
- *
- * @param date date string
- * @return parsed date, or <code>null</code> if the date can't be parsed
- * @see <a href="https://issues.apache.org/jira/browse/TIKA-495">TIKA-495</a>
- */
- private static synchronized Date parseDate(String date) {
- return DATE_UTILS.tryToParse(date);
+ /** Zone-less values read as UTC. */
+ private static Date parseDate(String date) {
+ Optional<TikaDates.ParsedDate> parsed = TikaDates.parse(date);
+ return parsed.isPresent() && parsed.get().isFullPrecision() ? Date.from(parsed.get().toInstant()) : null;
}
/**
@@ -275,6 +269,12 @@ public class Metadata implements Serializable {
/**
* Returns the value of the identified Date based metadata property. If many values are
* associated to the specified property, then the first one is returned.
+ * <p>
+ * The value is read with {@link org.apache.tika.utils.TikaDates}: a zone-less value is read
+ * as UTC (never the JVM default zone), a date-only {@code yyyy-MM-dd} value as midday UTC.
+ * Partial dates ({@code yyyy}, {@code yyyy-MM}) and years outside
+ * {@value org.apache.tika.utils.TikaDates#MIN_YEAR}..{@value org.apache.tika.utils.TikaDates#MAX_YEAR}
+ * return null.
*
* @param property simple date property definition
* @return property value as a Date, or <code>null</code> if the property is not set, or not
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/filter/DateNormalizingMetadataFilter.java b/tika-core/src/main/java/org/apache/tika/metadata/filter/DateNormalizingMetadataFilter.java
index e4f4b2aecf..02a749db54 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/filter/DateNormalizingMetadataFilter.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/filter/DateNormalizingMetadataFilter.java
@@ -16,11 +16,13 @@
*/
package org.apache.tika.metadata.filter;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
+import java.time.DateTimeException;
+import java.time.OffsetDateTime;
import java.time.ZoneId;
-import java.util.Date;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Locale;
+import java.util.Optional;
import java.util.TimeZone;
import org.slf4j.Logger;
@@ -31,13 +33,15 @@ import org.apache.tika.config.ConfigDeserializer;
import org.apache.tika.config.JsonConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
+import org.apache.tika.utils.TikaDates;
/**
- * Some dates in some file formats do not have a timezone.
- * Tika correctly stores these without a timezone, e.g. 'yyyy-MM-dd'T'HH:mm:ss'
- * This can be a problem if end points expect a 'Z' timezone.
- * This filter makes the assumption that dates without timezones are UTC
- * and always modifies the date to: "yyyy-MM-dd'T'HH:mm:ss'Z'"
+ * Rewrites every value of every DATE property to canonical UTC, "yyyy-MM-dd'T'HH:mm:ss'Z'",
+ * for end points that require a 'Z' timezone. An explicit offset is honored; zone-less values
+ * are read in the default time zone (UTC unless set), and date-only values as midday there.
+ * Partial dates are coerced: missing fields take their minimum, so "2019" becomes
+ * 2019-01-01 and "2019-06" becomes 2019-06-01, both at midday in the default zone.
+ * Unparseable values are left as they are.
*
* Users can specify an alternate defaultTimeZone with
* {@link DateNormalizingMetadataFilter#setDefaultTimeZone(String)} to apply
@@ -56,6 +60,9 @@ public class DateNormalizingMetadataFilter extends MetadataFilterBase {
private static TimeZone UTC = TimeZone.getTimeZone("UTC");
+ private static final DateTimeFormatter UTC_OUT =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+
private static final Logger LOGGER = LoggerFactory.getLogger(DateNormalizingMetadataFilter.class);
private TimeZone defaultTimeZone = UTC;
@@ -83,36 +90,45 @@ public class DateNormalizingMetadataFilter extends MetadataFilterBase {
}
protected void filter(Metadata metadata) {
- SimpleDateFormat dateFormatter = null;
- SimpleDateFormat utcFormatter = null;
for (String n : metadata.names()) {
-
Property property = Property.get(n);
- if (property != null) {
- if (property.getValueType().equals(Property.ValueType.DATE)) {
- String dateString = metadata.get(property);
- if (dateString.endsWith("Z")) {
- continue;
- }
- if (dateFormatter == null) {
- dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US);
- dateFormatter.setTimeZone(defaultTimeZone);
- utcFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
- utcFormatter.setTimeZone(UTC);
- }
- Date d = null;
- try {
- d = dateFormatter.parse(dateString);
- metadata.set(property, utcFormatter.format(d));
- } catch (ParseException e) {
- LOGGER.warn("Couldn't convert date to default time zone: >"
- + dateString + "<");
- }
+ if (property == null || !property.getValueType().equals(Property.ValueType.DATE)) {
+ continue;
+ }
+ String[] values = metadata.getValues(property);
+ for (int i = 0; i < values.length; i++) {
+ String utc = toUtc(values[i]);
+ if (utc != null) {
+ values[i] = utc;
}
}
+ metadata.set(property, values);
}
}
+ /** @return canonical UTC, or null to leave the value as is (unparseable) */
+ private String toUtc(String value) {
+ Optional<TikaDates.ParsedDate> parsed = TikaDates.parse(value);
+ if (parsed.isEmpty()) {
+ LOGGER.warn("Couldn't convert date to UTC: >{}<", abbreviate(value));
+ return null;
+ }
+ // a partial's LocalDateTime is already day 1 at midday
+ TikaDates.ParsedDate d = parsed.get();
+ try {
+ OffsetDateTime odt = d.hasZone() ? d.getLocalDateTime().atOffset(d.getOffset())
+ : d.getLocalDateTime().atZone(defaultTimeZone.toZoneId()).toOffsetDateTime();
+ return odt.withOffsetSameInstant(ZoneOffset.UTC).format(UTC_OUT);
+ } catch (DateTimeException e) {
+ LOGGER.warn("Couldn't convert date to UTC: >{}<", abbreviate(value));
+ return null;
+ }
+ }
+
+ private static String abbreviate(String value) {
+ return value.length() > 100 ? value.substring(0, 100) + "..." : value;
+ }
+
public void setDefaultTimeZone(String timeZoneId) {
this.defaultTimeZone = TimeZone.getTimeZone(ZoneId.of(timeZoneId));
}
diff --git a/tika-core/src/main/java/org/apache/tika/utils/DateUtils.java b/tika-core/src/main/java/org/apache/tika/utils/DateUtils.java
index 699ecad50b..97bdddc5ef 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/DateUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/DateUtils.java
@@ -16,15 +16,10 @@
*/
package org.apache.tika.utils;
-import java.text.DateFormat;
-import java.text.DateFormatSymbols;
-import java.text.SimpleDateFormat;
import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
-import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
@@ -50,22 +45,6 @@ public class DateUtils {
* default mapping would result in "2012-02-16" for UTC-8).
*/
public static final TimeZone MIDDAY = TimeZone.getTimeZone("GMT-12:00");
- /**
- * So we can return Date objects for these, this is the
- * list (in preference order) of the various ISO-8601
- * variants that we try when processing a date based
- * property.
- */
- private final List<DateFormat> iso8601InputFormats = loadDateFormats();
-
- private static DateFormat createDateFormat(String format, TimeZone timezone) {
- final SimpleDateFormat sdf = new SimpleDateFormat(format, new DateFormatSymbols(Locale.US));
- if (timezone != null) {
- sdf.setTimeZone(timezone);
- }
- return sdf;
- }
-
/**
* Returns a ISO 8601 representation of the given date in UTC,
* truncated to the seconds unit. This method is thread safe and non-blocking.
@@ -120,52 +99,9 @@ public class DateUtils {
return calendar.toInstant().truncatedTo(ChronoUnit.SECONDS).toString();
}
- private List<DateFormat> loadDateFormats() {
- List<DateFormat> dateFormats = new ArrayList<>();
- // yyyy-mm-ddThh...
- dateFormats.add(createDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", UTC)); // UTC/Zulu
- dateFormats.add(createDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", null)); // With timezone
- dateFormats.add(createDateFormat("yyyy-MM-dd'T'HH:mm:ss", null)); // Without timezone
- // yyyy-mm-dd hh...
- dateFormats.add(createDateFormat("yyyy-MM-dd' 'HH:mm:ss'Z'", UTC)); // UTC/Zulu
- dateFormats.add(createDateFormat("yyyy-MM-dd' 'HH:mm:ssZ", null)); // With timezone
- dateFormats.add(createDateFormat("yyyy-MM-dd' 'HH:mm:ss", null)); // Without timezone
- // Date without time, set to Midday UTC
- dateFormats.add(createDateFormat("yyyy-MM-dd", MIDDAY)); // Normal date format
- dateFormats.add(createDateFormat("yyyy:MM:dd",
- MIDDAY)); // Image (IPTC/EXIF) format
-
- return dateFormats;
- }
-
- /**
- * Tries to parse the date string; returns null if no parse was possible.
- * <p>
- * This is not thread safe! Wrap in synchronized or create new {@link DateUtils}
- * for each class.
- *
- * @param dateString
- * @return
- */
+ /** Delegates to {@link TikaDates}. */
public Date tryToParse(String dateString) {
- if (dateString == null || dateString.length() < 6) {
- return null;
- }
- // Java doesn't like timezones in the form ss+hh:mm
- // It only likes the hhmm form, without the colon
- int n = dateString.length();
- if (dateString.charAt(n - 3) == ':' &&
- (dateString.charAt(n - 6) == '+' || dateString.charAt(n - 6) == '-')) {
- dateString = dateString.substring(0, n - 3) + dateString.substring(n - 2);
- }
-
- for (DateFormat df : iso8601InputFormats) {
- try {
- return df.parse(dateString);
- } catch (java.text.ParseException e) {
- //swallow
- }
- }
- return null;
+ return TikaDates.parse(dateString).filter(TikaDates.ParsedDate::isFullPrecision)
+ .map(d -> Date.from(d.toInstant())).orElse(null);
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/utils/TikaDates.java b/tika-core/src/main/java/org/apache/tika/utils/TikaDates.java
new file mode 100644
index 0000000000..2284ddc20f
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/utils/TikaDates.java
@@ -0,0 +1,527 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.utils;
+
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.Month;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * The one parser for date strings from documents. Each value is routed by its leading shape
+ * to a family (ISO 8601 / W3C-DTF, PDF {@code D:}, EXIF, RFC 5322 and other month-name forms,
+ * ctime, slash dates, 8/12/14-digit strings) and must match that family whole; there is no
+ * prefix matching. Thread-safe.
+ * <p>
+ * Rules: years outside {@value #MIN_YEAR}..{@value #MAX_YEAR} and offsets outside
+ * -12:00..+14:00 are rejected; the calendar is proleptic Gregorian as written; slash dates are
+ * month-first unless the first field is over 12 (day-first) or over 31 (yy/M/d); two-digit
+ * years 00-49 are 20xx (RFC 5322); named zones are fixed offsets (EDT is -4 all year) and an
+ * unknown 2-5 letter name means the zone is unknown (RFC 5322 4.3); a date-only value has no
+ * zone even if one was written.
+ * <p>
+ * Producer quirks accepted: {@code HH:mmss}, {@code HH.mm.ss}, one-digit seconds, trailing
+ * {@code :cc} after seconds, {@code Z00:00}, apostrophes in offsets ({@code -06'00'}),
+ * AM/PM after an ISO time, {@code D:} followed by ISO, a trailing {@code (comment)}, a missing
+ * sign on a {@code 0000} offset, and, in mail forms only, junk after a complete time and
+ * numeric offset. Inputs over {@value #MAX_LENGTH} characters are rejected unread.
+ * <p>
+ * Storage ({@link ParsedDate#toMetadataString()}): explicit zone -> canonical UTC {@code ...Z};
+ * no zone -> {@code yyyy-MM-dd'T'HH:mm:ss}; date-only -> {@code yyyy-MM-dd}. Reading back
+ * ({@link ParsedDate#toInstant()}) treats no zone as UTC and date-only as midday UTC.
+ */
+public final class TikaDates {
+
+ public static final int MIN_YEAR = 1000;
+ public static final int MAX_YEAR = 2200;
+ // real dates, mail comments included, are far shorter; bounds regex work on hostile input
+ static final int MAX_LENGTH = 256;
+
+ public enum Precision { YEAR, MONTH, DAY, MINUTE, SECOND }
+
+ /** The offset is null when the source had none. */
+ public static final class ParsedDate {
+ private final LocalDateTime local;
+ private final ZoneOffset offset;
+ private final Precision precision;
+
+ ParsedDate(LocalDateTime local, ZoneOffset offset, Precision precision) {
+ this.local = local;
+ this.offset = offset;
+ this.precision = precision;
+ }
+
+ public LocalDateTime getLocalDateTime() {
+ return local;
+ }
+
+ public ZoneOffset getOffset() {
+ return offset;
+ }
+
+ public boolean hasZone() {
+ return offset != null;
+ }
+
+ public Precision getPrecision() {
+ return precision;
+ }
+
+ /** Day precision or finer. */
+ public boolean isFullPrecision() {
+ return precision.compareTo(Precision.DAY) >= 0;
+ }
+
+ /** Zone-less values read as UTC. */
+ public Instant toInstant() {
+ return local.toInstant(offset == null ? ZoneOffset.UTC : offset);
+ }
+
+ /** Canonical {@code yyyy-MM-dd'T'HH:mm:ss'Z'}, truncated to seconds. */
+ public String toCanonicalString() {
+ return CANONICAL.format(toInstant().truncatedTo(ChronoUnit.SECONDS).atOffset(ZoneOffset.UTC));
+ }
+
+ /** Stored form: {@code yyyy-MM-dd} if date-only, else {@code ...Z} if zoned, else {@code yyyy-MM-dd'T'HH:mm:ss}. */
+ public String toMetadataString() {
+ if (precision == Precision.DAY) {
+ return DATE_ONLY.format(local);
+ }
+ if (offset != null) {
+ return toCanonicalString();
+ }
+ return ZONELESS.format(local);
+ }
+
+ @Override
+ public String toString() {
+ return local + (offset == null ? "" : offset.toString()) + " (" + precision + ")";
+ }
+ }
+
+ private static final DateTimeFormatter CANONICAL =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ private static final DateTimeFormatter ZONELESS =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT);
+ private static final DateTimeFormatter DATE_ONLY = DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ROOT);
+
+ private static final String OFFSET = "(?:(?:GMT|UTC)?([+-])(\\d{1,2})(?::?(\\d{2}))?)";
+ private static final String ZONE = "(?:\\s*(?:" + OFFSET + "|(Z|[A-Z]{1,5})))?";
+ private static final String TIME =
+ "(\\d{1,2}):(\\d{1,2})(?::(\\d{1,2})(?:[.,]\\d{1,9})?)?(?:\\s*([AP])\\.?M\\.?)?";
+
+ // also producer quirks: HH:mmss, HH.mm.ss, 1-digit seconds, trailing :cc
+ private static final Pattern ISO = Pattern.compile("(\\d{4})(?:-(\\d{1,2})(?:-(\\d{1,2})"
+ + "(?:(?:T|\\s+)(\\d{1,2})([:.])(\\d{2})(?:(?:\\5|(?<=:\\d\\d))(\\d{1,2})(?::\\d{2})?(?:[.,]\\d{1,9})?)?"
+ + "(?:\\s*([AP])\\.?M\\.?)?)?)?)?" + ZONE);
+ private static final Pattern PDF = Pattern.compile("(D:)?\\s*(\\d{4})(\\d{2})?(\\d{2})?(\\d{2})?(\\d{2})?(\\d{2})?"
+ + "(?:(Z)(?:00'?00'?|')?|([+-])(\\d{1,2})(?:'?(\\d{2})'{0,2})?)?");
+ private static final Pattern EXIF = Pattern.compile("(\\d{4}):(\\d{2}):(\\d{2})"
+ + "(?:[ T](\\d{2}):(\\d{2})(?::(\\d{2})(?:\\.\\d{1,9})?)?)?" + ZONE);
+ // d MMM yyyy [time] [zone]
+ private static final Pattern DAY_MONTH = Pattern.compile(
+ "(\\d{1,2})[ -]([A-Z]{3,9})\\.?,?[ -](\\d{4}|\\d{2})(?:,?\\s+" + TIME + ")?" + ZONE);
+ // MMM d[,] yyyy [time] [zone]
+ private static final Pattern MONTH_DAY = Pattern.compile(
+ "([A-Z]{3,9})\\.?\\s+(\\d{1,2})(?:ST|ND|RD|TH)?,?\\s+(\\d{4})(?:,?\\s+" + TIME + ")?" + ZONE);
+ // ctime: MMM d HH:mm:ss [zone-name] yyyy [offset]
+ private static final Pattern CTIME = Pattern.compile(
+ "([A-Z]{3,9})\\s+(\\d{1,2})\\s+" + TIME + "(?:\\s+([A-Z]{1,5}))?\\s+(\\d{4})" + ZONE);
+ // M/d/yy[yy] or d/M/yy[yy] [time] [zone]
+ private static final Pattern SLASH = Pattern.compile(
+ "(\\d{1,2})/(\\d{1,2})/(\\d{4}|\\d{1,2})(?:,?\\s+" + TIME + ")?" + ZONE);
+ private static final Pattern SLASH_YMD = Pattern.compile(
+ "(\\d{4})/(\\d{1,2})/(\\d{1,2})(?:\\s+" + TIME + ")?" + ZONE);
+
+ private static final Pattern WEEKDAY = Pattern.compile(
+ "^(?:MON|TUE|TUES|WED|THU|THUR|THURS|FRI|SAT|SUN)(?:DAY|NESDAY|SDAY|RSDAY|URDAY)?\\.?,?\\s+");
+ private static final Pattern TRAILING_COMMENT = Pattern.compile("\\s*\\([^()]*\\)$");
+ private static final Pattern WHITESPACE = Pattern.compile("\\s+");
+ // producer quirk: an explicit UTC marker followed by a redundant zero offset
+ private static final Pattern Z_ZERO = Pattern.compile("Z00:?00");
+ // "-0400 EDT": a name after an explicit offset is redundant
+ private static final Pattern OFFSET_THEN_NAME = Pattern.compile("([+-]\\d{2}:?\\d{2})\\s+[A-Z]{2,5}$");
+ private static final Pattern AFTER_OFFSET = Pattern.compile("\\d:\\d{2}(?::\\d{2})?\\s*([+-]\\d{4})(?![\\d:]).+$");
+ // RFC 5322 allows a missing sign only for the zero offset
+ private static final Pattern UNSIGNED_ZERO = Pattern.compile("\\s0000$");
+
+ private static final Map<String, Integer> ZONE_NAMES = Map.ofEntries(
+ Map.entry("Z", 0), Map.entry("UT", 0), Map.entry("UTC", 0), Map.entry("GMT", 0),
+ Map.entry("EST", -5), Map.entry("EDT", -4), Map.entry("CST", -6), Map.entry("CDT", -5),
+ Map.entry("MST", -7), Map.entry("MDT", -6), Map.entry("PST", -8), Map.entry("PDT", -7),
+ Map.entry("CET", 1), Map.entry("CEST", 2), Map.entry("MET", 1), Map.entry("MEST", 2),
+ Map.entry("WET", 0), Map.entry("WEST", 1), Map.entry("EET", 2), Map.entry("EEST", 3),
+ Map.entry("MSK", 3), Map.entry("JST", 9), Map.entry("HST", -10), Map.entry("AKST", -9),
+ Map.entry("AKDT", -8), Map.entry("NZST", 12), Map.entry("NZDT", 13));
+
+ private static final String[] MONTHS = {"JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG",
+ "SEP", "OCT", "NOV", "DEC"};
+
+ private TikaDates() {
+ }
+
+ public static Optional<ParsedDate> parse(String raw) {
+ if (raw == null) {
+ return Optional.empty();
+ }
+ String trimmed = raw.trim();
+ if (trimmed.length() > MAX_LENGTH) {
+ return Optional.empty();
+ }
+ String s = WHITESPACE.matcher(trimmed).replaceAll(" ").toUpperCase(Locale.ROOT);
+ if (s.isEmpty()) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.ofNullable(route(s)).filter(TikaDates::inBounds);
+ } catch (DateTimeException | NumberFormatException e) {
+ return Optional.empty();
+ }
+ }
+
+ /** @return canonical UTC string for a full-precision date, else null */
+ public static String normalize(String raw) {
+ return parse(raw).filter(ParsedDate::isFullPrecision).map(ParsedDate::toCanonicalString).orElse(null);
+ }
+
+ /** @return {@link ParsedDate#toMetadataString()} for a full-precision date, else null */
+ public static String toMetadataString(String raw) {
+ return parse(raw).filter(ParsedDate::isFullPrecision).map(ParsedDate::toMetadataString).orElse(null);
+ }
+
+ private static boolean inBounds(ParsedDate d) {
+ return inYearBounds(d.toInstant());
+ }
+
+ /** Like {@link #toMetadataString(String)} but keeps {@code yyyy} / {@code yyyy-MM}; source-specific keys only. */
+ public static String toMetadataStringKeepPartial(String raw) {
+ return parse(raw).map(d -> {
+ LocalDateTime l = d.getLocalDateTime();
+ switch (d.getPrecision()) {
+ case YEAR:
+ return String.format(Locale.ROOT, "%04d", l.getYear());
+ case MONTH:
+ return String.format(Locale.ROOT, "%04d-%02d", l.getYear(), l.getMonthValue());
+ default:
+ return d.toMetadataString();
+ }
+ }).orElse(null);
+ }
+
+ /** For dates decoded from binary fields. */
+ public static boolean inYearBounds(Instant instant) {
+ if (instant == null) {
+ return false;
+ }
+ int y = instant.atOffset(ZoneOffset.UTC).getYear();
+ return y >= MIN_YEAR && y <= MAX_YEAR;
+ }
+
+ private static ParsedDate route(String s) {
+ s = TRAILING_COMMENT.matcher(s).replaceFirst("");
+ if (s.isEmpty()) {
+ return null;
+ }
+ char c = s.charAt(0);
+ if (c == 'D' && s.startsWith("D:")) {
+ return s.length() > 6 && s.charAt(6) == '-' ? iso(s.substring(2)) : pdf(s);
+ }
+ if (Character.isDigit(c)) {
+ if (s.length() > 4 && Character.isDigit(s.charAt(1)) && Character.isDigit(s.charAt(2))
+ && Character.isDigit(s.charAt(3))) {
+ char sep = s.charAt(4);
+ if (sep == '-') {
+ return iso(s);
+ }
+ if (sep == ':') {
+ return exif(s);
+ }
+ if (sep == '/') {
+ return slashYmd(s);
+ }
+ if (Character.isDigit(sep)) {
+ return pdf(s);
+ }
+ return null;
+ }
+ if (s.length() == 4) {
+ return iso(s);
+ }
+ if (s.indexOf('/') > 0 && s.indexOf('/') < 3) {
+ return slash(s);
+ }
+ return dayMonth(stripMailNoise(s));
+ }
+ String t = WEEKDAY.matcher(s).replaceFirst("");
+ if (t.isEmpty() || t.equals(s) && !Character.isLetter(c)) {
+ return null;
+ }
+ if (Character.isDigit(t.charAt(0))) {
+ return route(t);
+ }
+ t = stripMailNoise(t);
+ ParsedDate d = monthDay(t);
+ return d != null ? d : ctime(t);
+ }
+
+ private static String stripMailNoise(String s) {
+ Matcher junk = AFTER_OFFSET.matcher(s);
+ if (junk.find()) {
+ s = s.substring(0, junk.end(1));
+ }
+ s = TRAILING_COMMENT.matcher(s).replaceFirst("");
+ s = s.replace(" AT ", " ");
+ s = OFFSET_THEN_NAME.matcher(s).replaceFirst("$1");
+ return UNSIGNED_ZERO.matcher(s).replaceFirst(" +0000");
+ }
+
+ private static ParsedDate iso(String s) {
+ // PDF-style offsets in ISO strings: -06'00'
+ Matcher m = ISO.matcher(Z_ZERO.matcher(s.replace("'", "")).replaceFirst("Z"));
+
+ if (!m.matches()) {
+ return null;
+ }
+ int year = Integer.parseInt(m.group(1));
+ if (m.group(2) == null) {
+ return partial(year, 1, Precision.YEAR);
+ }
+ if (m.group(3) == null) {
+ return partial(year, Integer.parseInt(m.group(2)), Precision.MONTH);
+ }
+ return build(year, Integer.parseInt(m.group(2)), Integer.parseInt(m.group(3)),
+ m.group(4), m.group(6), m.group(7), m.group(8), zone(m, 9));
+ }
+
+ private static ParsedDate pdf(String s) {
+ Matcher m = PDF.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ boolean prefixed = m.group(1) != null;
+ int digits = 4;
+ for (int g = 3; g <= 7 && m.group(g) != null; g++) {
+ digits += 2;
+ }
+ // bare digits are dates only at 8, 12 or 14 digits
+ if (!prefixed && digits != 8 && digits != 12 && digits != 14) {
+ return null;
+ }
+ int year = Integer.parseInt(m.group(2));
+ if (m.group(3) == null) {
+ return partial(year, 1, Precision.YEAR);
+ }
+ if (m.group(4) == null) {
+ return partial(year, Integer.parseInt(m.group(3)), Precision.MONTH);
+ }
+ if (m.group(5) != null && m.group(6) == null) {
+ return null;
+ }
+ ZoneOffset off = null;
+ if (m.group(8) != null) {
+ off = ZoneOffset.UTC;
+ } else if (m.group(9) != null) {
+ off = offset(m.group(9), m.group(10), m.group(11));
+ }
+ return build(year, Integer.parseInt(m.group(3)), Integer.parseInt(m.group(4)),
+ m.group(5), m.group(6), m.group(7), null, new ZoneOffset[]{off});
+ }
+
+ private static ParsedDate exif(String s) {
+ Matcher m = EXIF.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ return build(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)), Integer.parseInt(m.group(3)),
+ m.group(4), m.group(5), m.group(6), null, zone(m, 7));
+ }
+
+ private static ParsedDate dayMonth(String s) {
+ Matcher m = DAY_MONTH.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ int month = month(m.group(2));
+ if (month < 0) {
+ return null;
+ }
+ return build(year(m.group(3)), month, Integer.parseInt(m.group(1)),
+ m.group(4), m.group(5), m.group(6), m.group(7), zone(m, 8));
+ }
+
+ private static ParsedDate monthDay(String s) {
+ Matcher m = MONTH_DAY.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ int month = month(m.group(1));
+ if (month < 0) {
+ return null;
+ }
+ return build(Integer.parseInt(m.group(3)), month, Integer.parseInt(m.group(2)),
+ m.group(4), m.group(5), m.group(6), m.group(7), zone(m, 8));
+ }
+
+ private static ParsedDate ctime(String s) {
+ Matcher m = CTIME.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ int month = month(m.group(1));
+ if (month < 0) {
+ return null;
+ }
+ ZoneOffset[] z = zone(m, 9);
+ if (z == null) {
+ return null;
+ }
+ if (m.group(7) != null) {
+ ZoneOffset[] named = namedOrUnknown(m.group(7));
+ if (named == null) {
+ return null;
+ }
+ if (z[0] == null) {
+ z = named;
+ } else if (named[0] != null && !named[0].equals(z[0])) {
+ return null; // a known name contradicting the numeric offset
+ }
+ }
+ return build(Integer.parseInt(m.group(8)), month, Integer.parseInt(m.group(2)),
+ m.group(3), m.group(4), m.group(5), m.group(6), z);
+ }
+
+ private static ParsedDate slash(String s) {
+ Matcher m = SLASH.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ int a = Integer.parseInt(m.group(1));
+ int b = Integer.parseInt(m.group(2));
+ if (a > 31) {
+ // only a year can be > 31: yy/M/d
+ return build(year(m.group(1)), b, Integer.parseInt(m.group(3)), m.group(4), m.group(5), m.group(6),
+ m.group(7), zone(m, 8));
+ }
+ if (m.group(3).length() == 1) {
+ return null;
+ }
+ int month = a > 12 ? b : a;
+ int day = a > 12 ? a : b;
+ return build(year(m.group(3)), month, day, m.group(4), m.group(5), m.group(6), m.group(7), zone(m, 8));
+ }
+
+ private static ParsedDate slashYmd(String s) {
+ Matcher m = SLASH_YMD.matcher(s);
+ if (!m.matches()) {
+ return null;
+ }
+ return build(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)), Integer.parseInt(m.group(3)),
+ m.group(4), m.group(5), m.group(6), m.group(7), zone(m, 8));
+ }
+
+ /** ZONE groups from {@code g}; null means reject. */
+ private static ZoneOffset[] zone(Matcher m, int g) {
+ if (m.group(g) != null) {
+ return new ZoneOffset[]{offset(m.group(g), m.group(g + 1), m.group(g + 2))};
+ }
+ if (m.group(g + 3) != null) {
+ return namedOrUnknown(m.group(g + 3));
+ }
+ return new ZoneOffset[]{null};
+ }
+
+ private static ZoneOffset named(String name) {
+ Integer h = ZONE_NAMES.get(name);
+ return h == null ? null : ZoneOffset.ofHours(h);
+ }
+
+ // RFC 5322 4.3: an unknown zone name means zone unknown
+ private static ZoneOffset[] namedOrUnknown(String name) {
+ ZoneOffset named = named(name);
+ if (named != null) {
+ return new ZoneOffset[]{named};
+ }
+ return name.length() >= 2 ? new ZoneOffset[]{null} : null;
+ }
+
+ private static ZoneOffset offset(String sign, String hh, String mm) {
+ int h = Integer.parseInt(hh);
+ int min = mm == null ? 0 : Integer.parseInt(mm);
+ int total = h * 60 + min;
+ if (min > 59 || ("-".equals(sign) ? total > 12 * 60 : total > 14 * 60)) {
+ throw new DateTimeException("offset out of range");
+ }
+ return "-".equals(sign) ? ZoneOffset.ofHoursMinutes(-h, -min) : ZoneOffset.ofHoursMinutes(h, min);
+ }
+
+ private static int month(String name) {
+ if (name.length() < 3) {
+ return -1;
+ }
+ String p = name.substring(0, 3);
+ for (int i = 0; i < MONTHS.length; i++) {
+ if (MONTHS[i].equals(p) && fullMonthName(i).startsWith(name)) {
+ return i + 1;
+ }
+ }
+ return -1;
+ }
+
+ private static String fullMonthName(int i) {
+ return Month.of(i + 1).name();
+ }
+
+ private static int year(String y) {
+ int v = Integer.parseInt(y);
+ if (y.length() == 2) {
+ return v < 50 ? 2000 + v : 1900 + v;
+ }
+ return v;
+ }
+
+ private static ParsedDate partial(int year, int month, Precision p) {
+ return new ParsedDate(LocalDateTime.of(year, month, 1, 12, 0), null, p);
+ }
+
+ private static ParsedDate build(int year, int month, int day, String hh, String mi, String ss,
+ String ampm, ZoneOffset[] zone) {
+ if (zone == null) {
+ return null;
+ }
+ if (hh == null) {
+ return new ParsedDate(LocalDateTime.of(year, month, day, 12, 0), null, Precision.DAY);
+ }
+ int hour = Integer.parseInt(hh);
+ if (ampm != null) {
+ if (hour < 1 || hour > 12) {
+ return null;
+ }
+ hour = hour % 12 + ("P".equals(ampm) ? 12 : 0);
+ }
+ int sec = ss == null ? 0 : Integer.parseInt(ss);
+ return new ParsedDate(LocalDateTime.of(year, month, day, hour, Integer.parseInt(mi), sec), zone[0],
+ ss == null ? Precision.MINUTE : Precision.SECOND);
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/metadata/MetadataZonelessDateTest.java b/tika-core/src/test/java/org/apache/tika/metadata/MetadataZonelessDateTest.java
new file mode 100644
index 0000000000..4174b697cf
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/metadata/MetadataZonelessDateTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.time.Instant;
+import java.util.TimeZone;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Isolated;
+import org.junit.jupiter.api.parallel.ResourceLock;
+import org.junit.jupiter.api.parallel.Resources;
+
+@Isolated
+@ResourceLock(Resources.TIME_ZONE)
+public class MetadataZonelessDateTest {
+
+ private static TimeZone originalTimeZone;
+
+ @BeforeAll
+ static void init() {
+ originalTimeZone = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Kiritimati"));
+ }
+
+ @AfterAll
+ static void tearDown() {
+ TimeZone.setDefault(originalTimeZone);
+ }
+
+ @Test
+ public void testGetDateIgnoresJvmZone() {
+ assertEquals(Instant.parse("2010-01-10T13:00:19Z"), getDate("2010-01-10T13:00:19"));
+ assertEquals(Instant.parse("2010-01-10T12:00:00Z"), getDate("2010-01-10"));
+ assertEquals(Instant.parse("2010-01-10T13:00:19Z"), getDate("2010-01-10T13:00:19Z"));
+ assertEquals(Instant.parse("2010-01-10T12:00:19Z"), getDate("2010-01-10T13:00:19+01:00"));
+ }
+
+ private static Instant getDate(String stored) {
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.CREATED, stored);
+ return m.getDate(TikaCoreProperties.CREATED).toInstant();
+ }
+
+ @Test
+ public void testGarbageIsNull() {
+ Metadata m = new Metadata();
+ for (String garbage : new String[]{"0-00-00T00:00:00Z", "0000-00-00", "2019", "INVALID"}) {
+ m.set(TikaCoreProperties.CREATED, garbage);
+ assertNull(m.getDate(TikaCoreProperties.CREATED), garbage);
+ assertNull(new org.apache.tika.utils.DateUtils().tryToParse(garbage), garbage);
+ }
+ }
+}
diff --git a/tika-core/src/test/java/org/apache/tika/utils/TikaDatesTest.java b/tika-core/src/test/java/org/apache/tika/utils/TikaDatesTest.java
new file mode 100644
index 0000000000..17bfed84d7
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/utils/TikaDatesTest.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.TimeZone;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.parallel.Isolated;
+import org.junit.jupiter.api.parallel.ResourceLock;
+import org.junit.jupiter.api.parallel.Resources;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+/** Runs in a +14h zone to catch JVM-zone leaks. */
+@Isolated
+@ResourceLock(Resources.TIME_ZONE)
+public class TikaDatesTest {
+
+ private static TimeZone originalTimeZone;
+
+ @BeforeAll
+ static void init() {
+ originalTimeZone = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Kiritimati"));
+ }
+
+ @AfterAll
+ static void tearDown() {
+ TimeZone.setDefault(originalTimeZone);
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @CsvSource(delimiter = '|', quoteCharacter = '"', value = {
+ // ISO / W3C-DTF (the bulk of XMP)
+ "2006-07-15T00:01:46+01:00 | 2006-07-14T23:01:46Z",
+ "2004-08-09T14:03:21-05:00 | 2004-08-09T19:03:21Z",
+ "2011-01-01T12:41:36Z | 2011-01-01T12:41:36Z",
+ "2007-01-01T00:07+01:00 | 2006-12-31T23:07:00Z",
+ "2010-03-08T21:37-05:00 | 2010-03-09T02:37:00Z",
+ "2016-08-26T07:06Z | 2016-08-26T07:06:00Z",
+ "2010-01-10T13:00:19 | 2010-01-10T13:00:19Z",
+ "2013-09-08T04:14:06.006 | 2013-09-08T04:14:06Z",
+ "2018-08-06T20:53:57.9843081+01:00 | 2018-08-06T19:53:57Z",
+ "2011-12-17T21:47:07.88+09:00 | 2011-12-17T12:47:07Z",
+ "2007-02-21T09:10:07.0+2:00 | 2007-02-21T07:10:07Z",
+ "2018-03-15T10:18:57+1:00 | 2018-03-15T09:18:57Z",
+ "2011-12-31T15:37:24.00986+03:30 | 2011-12-31T12:07:24Z",
+ "2010-05-09T21:34:38+0200 | 2010-05-09T19:34:38Z",
+ "2003-01-01 12:00:00 +0100 | 2003-01-01T11:00:00Z",
+ "2010-04-22 15:20 -0400 | 2010-04-22T19:20:00Z",
+ "2015-08-26T09:39 | 2015-08-26T09:39:00Z",
+ // date-only -> midday UTC
+ "2005-11-03 | 2005-11-03T12:00:00Z",
+ "2017-06-1 | 2017-06-01T12:00:00Z",
+ "1969-7-20 | 1969-07-20T12:00:00Z",
+ // PDF D: and compact
+ "D:20041028184621 | 2004-10-28T18:46:21Z",
+ "D:20080528153425+02'00' | 2008-05-28T13:34:25Z",
+ "D:20030101120000+05'30' | 2003-01-01T06:30:00Z",
+ "D:20030101120000+05'30 | 2003-01-01T06:30:00Z",
+ "D:20030101120000-08'00' | 2003-01-01T20:00:00Z",
+ "D:20030101120000Z | 2003-01-01T12:00:00Z",
+ "D:20030101120000Z00'00' | 2003-01-01T12:00:00Z",
+ "D:200301011200 | 2003-01-01T12:00:00Z",
+ "D:20041109 | 2004-11-09T12:00:00Z",
+ "\" D:20030101 \" | 2003-01-01T12:00:00Z",
+ "20030101120000 | 2003-01-01T12:00:00Z",
+ // EXIF
+ "2014:08:21 13:33:53 | 2014-08-21T13:33:53Z",
+ "2015:06:12 | 2015-06-12T12:00:00Z",
+ "2017:12:23 12:51:20Z | 2017-12-23T12:51:20Z",
+ // RFC 5322 (mail)
+ "\"Fri, 01 Dec 2006 00:06:00 GMT\" | 2006-12-01T00:06:00Z",
+ "\"Fri, 03 Dec 1999 15:17:26 -0500 (EST)\" | 1999-12-03T20:17:26Z",
+ "01 Aug 2001 11:14:22 -0400 | 2001-08-01T15:14:22Z",
+ "1 Apr 1999 11:40:10 -0500 | 1999-04-01T16:40:10Z",
+ "\"Fri, 16 Apr 93 08:25:04 -0400\" | 1993-04-16T12:25:04Z",
+ "\"Fri, 4 Sep 98 00:24:11 -0600\" | 1998-09-04T06:24:11Z",
+ "\"Fri, 01 Apr 1994 16:57 -0800 (PST)\" | 1994-04-02T00:57:00Z",
+ "\"Fri, 01 Dec 2000 08:39:07\" | 2000-12-01T08:39:07Z",
+ "\"Fri, 18 Jun 1993 9:42:24 -0400 (EDT)\" | 1993-06-18T13:42:24Z",
+ "10 Jul 1996 13:23 EDT | 1996-07-10T17:23:00Z",
+ "\"Fri, 1 Feb 91 18:33:59 PST\" | 1991-02-02T02:33:59Z",
+ "\"Thu, 6 May 2004 10:12:09 +0200 (CEST)\" | 2004-05-06T08:12:09Z",
+ "\"Fri, 12 Apr 2002 18:56:49 -0400 (Eastern Daylight Time)\" | 2002-04-12T22:56:49Z",
+ "\"Fri, 19 Oct 2007 10:23:52 -0600 (GMT-06:00)\" | 2007-10-19T16:23:52Z",
+ "\"Fri, 16 Jan 2004 10:45:33 0000\" | 2004-01-16T10:45:33Z",
+ "15 Nov 94 09:44:30 -5 | 1994-11-15T14:44:30Z",
+ "\"Thursday, 18 December 1997 11:31pm\" | 1997-12-18T23:31:00Z",
+ "29 Jan 2014 | 2014-01-29T12:00:00Z",
+ "9 November 2001 | 2001-11-09T12:00:00Z",
+ "\"3 March, 2017\" | 2017-03-03T12:00:00Z",
+ // US month-name forms
+ "\"Friday, April 02, 1999 1:20:07 PM\" | 1999-04-02T13:20:07Z",
+ "\"Fri, Nov 15, 1996 6:58 AM\" | 1996-11-15T06:58:00Z",
+ "\"Friday, December 12, 2008, 11:28 AM\" | 2008-12-12T11:28:00Z",
+ "\"Fri, May 11, 2012 at 6:06 PM\" | 2012-05-11T18:06:00Z",
+ "\"Monday, August 12, 1996 5:13PM\" | 1996-08-12T17:13:00Z",
+ "\"Fri, Mar 9, 2012 20:23\" | 2012-03-09T20:23:00Z",
+ "\"Monday, October 29, 2001 10:13\" | 2001-10-29T10:13:00Z",
+ "\"Friday, June 2, 2000 3:45 PM PDT\" | 2000-06-02T22:45:00Z",
+ "\"Friday, February 19 2010 13:31:33 CET\" | 2010-02-19T12:31:33Z",
+ "\"August 12, 1999\" | 1999-08-12T12:00:00Z",
+ "\"April 5, 1989\" | 1989-04-05T12:00:00Z",
+ "Aug 13 2008 | 2008-08-13T12:00:00Z",
+ "Fri Jan 18 2008 | 2008-01-18T12:00:00Z",
+ "Wed May 23 2018 17:52:58 GMT+0200 | 2018-05-23T15:52:58Z",
+ // ctime
+ "Fri Apr 10 04:00:33 PDT 1992 | 1992-04-10T11:00:33Z",
+ "Fri Apr 2 04:00:22 PST 1999 | 1999-04-02T12:00:22Z",
+ "Fri Aug 01 10:21:01 2014 -0700 | 2014-08-01T17:21:01Z",
+ "Fri Apr 20 13:38:50 2007 | 2007-04-20T13:38:50Z",
+ "THU JAN 01 22:51:33 1970 | 1970-01-01T22:51:33Z",
+ // slash: month-first, day-first when the first field > 12
+ "5/12/2008 | 2008-05-12T12:00:00Z",
+ "12/31/2008 13:14:15 | 2008-12-31T13:14:15Z",
+ "23/12/2009 16:52:18 | 2009-12-23T16:52:18Z",
+ "13/09/2016 | 2016-09-13T12:00:00Z",
+ "2/22/2016 4:00:00 PM | 2016-02-22T16:00:00Z",
+ "\"09/09/2018, 20:47\" | 2018-09-09T20:47:00Z",
+ "08/14/18 | 2018-08-14T12:00:00Z",
+ "5/4/05 | 2005-05-04T12:00:00Z",
+ "17/4/13 14:10 | 2013-04-17T14:10:00Z",
+ "2016/01/15 14:27:04 | 2016-01-15T14:27:04Z",
+ "2009/8/26 | 2009-08-26T12:00:00Z",
+ "96/1/02 | 1996-01-02T12:00:00Z", // first field > 31: yy/M/d
+ "96/12/2 | 1996-12-02T12:00:00Z",
+ // producer quirks
+ "2017-11-20T13:5836 | 2017-11-20T13:58:36Z",
+ "2018-09-13T18.20.16Z | 2018-09-13T18:20:16Z",
+ "2006-06-19T12:37:3Z | 2006-06-19T12:37:03Z",
+ "2012-08-02T19:22:29:93 | 2012-08-02T19:22:29Z",
+ "2008-01-14T19:43:09Z00:00 | 2008-01-14T19:43:09Z",
+ "2016-01-14T11:27:45-6':00 | 2016-01-14T17:27:45Z",
+ "2017-12-31 07:09:59 PM | 2017-12-31T19:09:59Z",
+ "D:2018-08-02T16:06:22+02'00' | 2018-08-02T14:06:22Z",
+ // PDF quirks
+ "D:20160308221458Z' | 2016-03-08T22:14:58Z",
+ "D:20210712135012+02'00'' | 2021-07-12T11:50:12Z",
+ "D:20190221101345-6'00' | 2019-02-21T16:13:45Z",
+ "8/11/2008 14:8:23 | 2008-08-11T14:08:23Z",
+ "\"Wednesday, 16/02/2011\" | 2011-02-16T12:00:00Z",
+ "2006-10-18 16:01:00 -0700 (Wed, 18 Oct 2006) | 2006-10-18T23:01:00Z",
+ "\"Mon, 29 Oct 2007 20:20:52 -0500From: someone\" | 2007-10-30T01:20:52Z",
+ "\"Thu, 27 Aug 1998 08:39:32 +0200 <br>\" | 1998-08-27T06:39:32Z",
+ "\"Sat, 19 Jun 93 16:34:27 JST\" | 1993-06-19T07:34:27Z",
+ "\"Wed, 16 Aug 2000 4:19:1 GMT\" | 2000-08-16T04:19:01Z",
+ // ambiguous zone name -> zone unknown
+ "15 Oct 99 13:47:26 BST | 1999-10-15T13:47:26Z",
+ "2010-07-28T11:02:12.000CEST | 2010-07-28T09:02:12Z",
+ "7-JUL-1994 02:36:51.00 | 1994-07-07T02:36:51Z",
+ "\"Tuesday, 04-Jul-95 06:04:28 GMT\" | 1995-07-04T06:04:28Z",
+ "\"April 18th, 2000\" | 2000-04-18T12:00:00Z",
+ "\"Nov. 21, 1998\" | 1998-11-21T12:00:00Z",
+ })
+ public void testNormalize(String raw, String expected) {
+ assertEquals(expected, TikaDates.normalize(raw));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @CsvSource(delimiter = '|', quoteCharacter = '"', value = {
+ "0-01-01T00:00:00Z", // year 0 -> -0001-12-30 (DateConverter)
+ "0-00-00T00:00:00Z", // -> -0001-11-28 (DateUtils)
+ "0000-01-01T00:00:00Z",
+ "0-00-00",
+ "9999-12-31", // sentinel, above MAX_YEAR
+ "9999/12/31",
+ "10000-01-01T00:59:59+01:00",
+ "406-04-16T07:15:00Z",
+ "2/23/201", // 3-digit year -> 0201 (DateConverter)
+ "00:15:54", // time only -> year 1 (DateUtils)
+ "2:16:58",
+ "2014-07-18T05:44:470Z", // overflow -> 05:51:50 (DateConverter)
+ "2015:06:12 10:11:",
+ "2016-13-01",
+ "2016-02-30",
+ "23/13/2009", // both fields > 12
+ "5/12/8", // 1-digit year
+ "18/02/98 25:00",
+ "11658090",
+ "137912",
+ "1057298",
+ "0",
+ "+02:00",
+ "()",
+ "CPY Document Creation Date",
+ "\"Mon, 7 Jan 2008 08:47:01 0100 (CET)\"", // unsigned non-zero offset: ambiguous
+ "10 Jun 1998 09:07:34 U",
+ "December 15, 2005 Report No. LOXA05-004",
+ "2015-09-28T04:47:26-16:00", // offsets outside -12..+14
+ "2017-09-12T02:10:54-21:00",
+ "2015-09-28T17:25:50+17:00",
+ "2017-05-11T19:05:84", // second 84 -> rolled to 19:06:24 (DateConverter)
+ "D:20080702212348+20'00'", // offset out of range
+ "D:2012010510394209'00'", // 16 digits, unsigned offset: ambiguous
+ "D:191010111144159",
+ "D:20170112095532--4'00'",
+ })
+ public void testRejected(String raw) {
+ assertNull(TikaDates.normalize(raw), raw);
+ }
+
+ @Test
+ public void testMailRegressionSuite() {
+ for (String s : new String[]{
+ "Mon, 9 May 16 01:32:00 GMT", "9 May 16 01:32:00 GMT", "Monday, 9 May 16 01:32:00 GMT",
+ "Mon, 9 May 2016 01:32:00 UTC", "9 May 2016 01:32:00 UTC", "09 May 2016 01:32:00 UTC",
+ "Mon, 9 May 2016 01:32:00Z", "Mon, 9 May 2016 01:32:00 Z", "Mon, 9 May 2016 01:32:00 GMT",
+ "Mon, 9 May 2016 01:32:00GMT", "Mon, 9 May 2016 01:32:00UTC",
+ "Mon, 9 May 2016 3:32:00 GMT+0200", "Mon, 9 May 2016 3:32:00 UTC+0200",
+ "Mon, 9 May 2016 7:32:00 UTC+0600 (BST)", " 9 May 2016 3:32:00 +0200",
+ " Mon, 9 May 2016 3:32:00 +0200", "Mon, 9 May 2016 3:32:00 +02:00", "9 May 2016 3:32:00 +02:00",
+ "Mon, 9 May 2016 3:32:00+02:00", "Mon, 9 May 2016 3:32:00+0200", " Sun, 8 May 2016 21:32:00 EDT"}) {
+ assertEquals("2016-05-09T01:32:00Z", TikaDates.normalize(s), s);
+ }
+ for (String s : new String[]{"Mon, 9 May 2016 01:32:00", "Monday, 9 May 2016 1:32 AM", "May 9 2016 1:32am",
+ "May 9 2016 1:32 am", "2016-05-09 01:32:00"}) {
+ assertEquals("2016-05-09T01:32:00Z", TikaDates.normalize(s), s);
+ assertEquals("2016-05-09T01:32:00", TikaDates.toMetadataString(s), s);
+ }
+ for (String s : new String[]{"May 15, 2016", "Sun, 15 May 2016", "15 May 2016", "2016-05-15"}) {
+ assertEquals("2016-05-15T12:00:00Z", TikaDates.normalize(s), s);
+ }
+ // named zones are fixed offsets (RFC 5322)
+ assertEquals("2016-05-09T02:32:00Z", TikaDates.normalize("Sun, 8 May 2016 21:32:00 EST"));
+ // must land in 1980..2010
+ for (String s : new String[]{"11/14/08", "1/14/08", "1/2/08", "12/1/2008", "12/02/1996", "96/1/02",
+ "96/12/02", "96/12/2", "1996/12/02", "Mon, 29 Jan 96 14:02 GMT", "7/20/95 1:12PM",
+ "08/14/2000 12:48 AM", "8/4/2000 1:48 AM", "8/1/03", "Wed, 27 Dec 95 11:20:40 EST",
+ "26 Aug 00 11:14:52 EDT"}) {
+ String n = TikaDates.normalize(s);
+ assertTrue(n != null && n.compareTo("1980") > 0 && n.compareTo("2010") < 0, s + " -> " + n);
+ }
+ // hyphenated year is not an offset
+ assertNull(TikaDates.normalize("10-10-2022 not a date"));
+ }
+
+ @Test
+ public void testKeepPartial() {
+ assertEquals("2018", TikaDates.toMetadataStringKeepPartial("2018"));
+ assertEquals("2018-05", TikaDates.toMetadataStringKeepPartial("2018-05"));
+ assertEquals("2018-05-01", TikaDates.toMetadataStringKeepPartial("2018-05-01"));
+ for (String junk : new String[]{"", " ", "0", "18", "0312", "2018 2018", "[www.example.com]"}) {
+ assertNull(TikaDates.toMetadataStringKeepPartial(junk), junk);
+ }
+ }
+
+ @Test
+ public void testInYearBounds() {
+ assertTrue(TikaDates.inYearBounds(java.time.Instant.parse("2012-02-20T16:44:22Z")));
+ assertFalse(TikaDates.inYearBounds(java.time.Instant.parse("0004-12-31T23:00:00Z")));
+ assertFalse(TikaDates.inYearBounds(java.time.Instant.parse("+31135-02-07T00:18:29Z")));
+ assertFalse(TikaDates.inYearBounds(null));
+ }
+
+ @Test
+ public void testEmptyAndNull() {
+ assertNull(TikaDates.normalize(null));
+ assertNull(TikaDates.normalize(""));
+ assertNull(TikaDates.normalize(" "));
+ assertFalse(TikaDates.parse(null).isPresent());
+ }
+
+ @Test
+ public void testPartialDates() {
+ assertEquals(TikaDates.Precision.YEAR, TikaDates.parse("2019").get().getPrecision());
+ assertEquals(TikaDates.Precision.MONTH, TikaDates.parse("2019-06").get().getPrecision());
+ assertEquals(TikaDates.Precision.YEAR, TikaDates.parse("D:2003").get().getPrecision());
+ assertFalse(TikaDates.parse("2019").get().isFullPrecision());
+ assertNull(TikaDates.normalize("2019"));
+ assertNull(TikaDates.normalize("2019-06"));
+ assertNull(TikaDates.normalize("D:2003"));
+ }
+
+ @Test
+ public void testPrecisionAndZone() {
+ TikaDates.ParsedDate d = TikaDates.parse("2010-01-10T13:00:19").get();
+ assertEquals(TikaDates.Precision.SECOND, d.getPrecision());
+ assertFalse(d.hasZone());
+ assertTrue(TikaDates.parse("2010-01-10T13:00:19Z").get().hasZone());
+ assertEquals(TikaDates.Precision.MINUTE, TikaDates.parse("2016-08-26T07:06Z").get().getPrecision());
+ assertEquals(TikaDates.Precision.DAY, TikaDates.parse("2005-11-03").get().getPrecision());
+ assertTrue(TikaDates.parse("2005-11-03").get().isFullPrecision());
+ }
+
+ @Test
+ public void testRoundTrip() {
+ for (String s : new String[]{"2006-07-15T00:01:46+01:00", "D:20080528153425+02'00'",
+ "Fri, 01 Dec 2006 00:06:00 GMT", "2005-11-03", "2014:08:21 13:33:53"}) {
+ String n = TikaDates.normalize(s);
+ assertEquals(n, TikaDates.normalize(n), s);
+ }
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @CsvSource(delimiter = '|', quoteCharacter = '"', value = {
+ "2006-07-15T00:01:46+01:00 | 2006-07-14T23:01:46Z",
+ "2016-08-26T07:06Z | 2016-08-26T07:06:00Z",
+ "2010-01-10T13:00:19 | 2010-01-10T13:00:19",
+ "2013-09-08T04:14:06.006 | 2013-09-08T04:14:06",
+ "2015-08-26T09:39 | 2015-08-26T09:39:00",
+ "D:20041028184621 | 2004-10-28T18:46:21",
+ "D:20080528153425+02'00' | 2008-05-28T13:34:25Z",
+ "2014:08:21 13:33:53 | 2014-08-21T13:33:53",
+ "\"Fri, 01 Dec 2000 08:39:07\" | 2000-12-01T08:39:07",
+ "\"Fri, 01 Dec 2006 00:06:00 GMT\" | 2006-12-01T00:06:00Z",
+ "2005-11-03 | 2005-11-03",
+ "2015:06:12 | 2015-06-12",
+ "D:20041109 | 2004-11-09",
+ "\"August 12, 1999\" | 1999-08-12",
+ })
+ public void testToMetadataString(String raw, String expected) {
+ assertEquals(expected, TikaDates.toMetadataString(raw));
+ }
+
+ @Test
+ public void testMetadataStringRoundTrip() {
+ for (String s : new String[]{"2006-07-15T00:01:46+01:00", "2010-01-10T13:00:19", "2015-08-26T09:39",
+ "D:20041028184621", "2014:08:21 13:33:53", "2005-11-03", "Fri, 01 Dec 2006 00:06:00 GMT"}) {
+ TikaDates.ParsedDate a = TikaDates.parse(s).get();
+ String stored = a.toMetadataString();
+ TikaDates.ParsedDate b = TikaDates.parse(stored).get();
+ assertEquals(a.toInstant(), b.toInstant(), s);
+ assertEquals(a.hasZone(), b.hasZone(), s);
+ assertEquals(a.getPrecision() == TikaDates.Precision.DAY, b.getPrecision() == TikaDates.Precision.DAY, s);
+ if (!a.hasZone()) {
+ assertEquals(a.getLocalDateTime(), b.getLocalDateTime(), s);
+ }
+ assertEquals(stored, TikaDates.toMetadataString(stored), s);
+ }
+ }
+
+ @Test
+ public void testMetadataStringPartialAndGarbage() {
+ assertNull(TikaDates.toMetadataString("2019"));
+ assertNull(TikaDates.toMetadataString("2019-06"));
+ assertNull(TikaDates.toMetadataString("0-01-01T00:00:00Z"));
+ assertNull(TikaDates.toMetadataString(null));
+ }
+
+ // the trailing-comment strip was quadratic: 80k '(' took tens of seconds per value
+ @Test
+ @Timeout(5)
+ public void testHostileLengthIsCheap() {
+ for (String prefix : new String[]{"", "1", "MON ", "JAN "}) {
+ String hostile = prefix + "(".repeat(100_000);
+ assertTrue(TikaDates.parse(hostile).isEmpty());
+ assertTrue(TikaDates.parse(hostile + ")X").isEmpty());
+ }
+ assertTrue(TikaDates.parse("2020-01-01T10:00:00Z" + " ".repeat(1000)).isPresent());
+ }
+
+ // an explicit UTC marker with a redundant zero offset is still a zone
+ @Test
+ public void testZeroOffsetQuirkKeepsZone() {
+ assertEquals("2020-01-01T10:00:00Z", TikaDates.toMetadataString("2020-01-01T10:00:00Z00:00"));
+ assertEquals("2020-01-01T10:00:00Z", TikaDates.toMetadataString("2020-01-01T10:00:00Z0000"));
+ assertTrue(TikaDates.parse("2020-01-01T10:00:00Z00:00").get().hasZone());
+ }
+
+ @Test
+ public void testCtimeZoneNames() {
+ // an unknown name means zone unknown, as in the other families
+ assertEquals("2020-01-01T10:00:00", TikaDates.toMetadataString("Wed Jan 1 10:00:00 IST 2020"));
+ assertEquals("2020-01-01T15:00:00Z", TikaDates.toMetadataString("Wed Jan 1 10:00:00 EST 2020"));
+ assertEquals("2020-01-01T15:00:00Z", TikaDates.toMetadataString("Wed Jan 1 10:00:00 EST 2020 -0500"));
+ assertNull(TikaDates.toMetadataString("Wed Jan 1 10:00:00 EST 2020 +0100"));
+ }
+
+ @Test
+ public void testOffsetRangeIncludesMinutes() {
+ assertEquals("2019-12-31T20:00:00Z", TikaDates.normalize("2020-01-01T10:00:00+14:00"));
+ assertEquals("2020-01-01T22:00:00Z", TikaDates.normalize("2020-01-01T10:00:00-12:00"));
+ assertEquals("2020-01-01T04:15:00Z", TikaDates.normalize("2020-01-01T10:00:00+05:45"));
+ assertNull(TikaDates.normalize("2020-01-01T10:00:00+14:45"));
+ assertNull(TikaDates.normalize("2020-01-01T10:00:00-12:30"));
+ }
+
+ // an offset on a bare date would move it to another day; date-only values have no zone
+ @Test
+ public void testDateOnlyIgnoresOffset() {
+ assertEquals("2020-01-01", TikaDates.toMetadataString("2020-01-01+14:00"));
+ assertEquals("2020-01-01T12:00:00Z", TikaDates.normalize("2020-01-01+14:00"));
+ assertFalse(TikaDates.parse("2020-01-01-12:00").get().hasZone());
+ }
+}
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index fa4cedacd4..062844af0b 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -281,7 +281,7 @@
</contributors>
<properties>
- <revision>4.1.1-SNAPSHOT</revision>
+ <revision>4.2.0-SNAPSHOT</revision>
<javaVersion>17</javaVersion>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
index a13030cc32..a415eea446 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/pom.xml
@@ -63,7 +63,6 @@
<module>tika-parser-jdbc-commons</module>
<module>tika-parser-datauri-commons</module>
<module>tika-parser-digest-commons</module>
- <module>tika-parser-mail-commons</module>
<module>tika-parser-xmp-commons</module>
<module>tika-parser-zip-commons</module>
@@ -81,6 +80,7 @@
<module>tika-parser-apple-module</module>
<module>tika-parser-cad-module</module>
<module>tika-parser-mail-module</module>
+ <module>tika-parser-mail-commons</module>
<module>tika-parser-miscoffice-module</module>
<module>tika-parser-news-module</module>
<module>tika-parser-crypto-module</module>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
index 1aeabe89e8..3d3d5b5677 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java
@@ -44,6 +44,7 @@ import org.apache.tika.parser.audio.NumberAndTotal;
import org.apache.tika.parser.mp3.ID3Tags.ID3Comment;
import org.apache.tika.parser.mp3.ID3Tags.ID3Picture;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.TikaDates;
/**
* The <code>Mp3Parser</code> is used to parse ID3 Version 1 Tag information
@@ -225,7 +226,7 @@ public class Mp3Parser implements Parser {
metadata.set(XMPDM.COPYRIGHT, tag.getCopyright());
metadata.set(XMPDM.ALBUM, tag.getAlbum());
metadata.set(XMPDM.COMPILATION, tag.getCompilation());
- metadata.set(XMPDM.RELEASE_DATE, tag.getYear());
+ setReleaseDate(metadata, tag.getYear());
metadata.set(XMPDM.GENRE, tag.getGenre());
for (ID3Comment comment : tag.getComments()) {
@@ -371,4 +372,12 @@ public class Mp3Parser implements Parser {
}
}
+
+ // ID3 years are free text: empty, "2018 2018", URLs
+ private static void setReleaseDate(Metadata metadata, String raw) {
+ String value = TikaDates.toMetadataStringKeepPartial(raw);
+ if (value != null) {
+ metadata.set(XMPDM.RELEASE_DATE, value);
+ }
+ }
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java
index d63b4edeae..5829b4ea66 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java
@@ -39,6 +39,7 @@ import org.apache.tika.metadata.XMP;
import org.apache.tika.metadata.XMPDM;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.TikaDates;
public class TikaUserDataBox {
@@ -310,7 +311,10 @@ public class TikaUserDataBox {
break;
case "\u00A9day" :
//this can be a year "2008" or a date "2017-04-26T07:00:00Z"
- metadata.set(XMPDM.RELEASE_DATE, value);
+ String releaseDate = TikaDates.toMetadataStringKeepPartial(value);
+ if (releaseDate != null) {
+ metadata.set(XMPDM.RELEASE_DATE, releaseDate);
+ }
xhtml.element("p", value);
break;
case "\u00A9cmt" :
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
index 4d13d4eabf..72d419dd55 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/ogg/OggAudioParser.java
@@ -56,6 +56,7 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.audio.CoverArt;
import org.apache.tika.parser.audio.NumberAndTotal;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.TikaDates;
/**
* Parent parser for the various Ogg Audio formats, such as
@@ -140,7 +141,10 @@ public abstract class OggAudioParser extends AbstractParser {
metadata.set(XMPDM.ARTIST, artist);
metadata.set(XMPDM.ALBUM, album);
metadata.set(XMPDM.GENRE, first(fields, VorbisComments.KEY_GENRE));
- metadata.set(XMPDM.RELEASE_DATE, first(fields, VorbisComments.KEY_DATE));
+ String releaseDate = TikaDates.toMetadataStringKeepPartial(first(fields, VorbisComments.KEY_DATE));
+ if (releaseDate != null) {
+ metadata.set(XMPDM.RELEASE_DATE, releaseDate);
+ }
metadata.add(XMP.CREATOR_TOOL, comments.getVendor());
metadata.add(VORBIS_VENDOR, comments.getVendor());
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
index b89f107302..3857c49fc5 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-font-module/src/main/java/org/apache/tika/parser/font/TrueTypeParser.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser.font;
import java.io.IOException;
+import java.util.Calendar;
import java.util.Collections;
import java.util.Set;
@@ -41,6 +42,7 @@ import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.TikaDates;
/**
* Parser for TrueType font files (TTF).
@@ -83,8 +85,15 @@ public class TrueTypeParser implements Parser {
// Report the details of the font
metadata.set(HttpHeaders.CONTENT_TYPE, TYPE.toString());
- metadata.set(TikaCoreProperties.CREATED, font.getHeader().getCreated());
- metadata.set(TikaCoreProperties.MODIFIED, font.getHeader().getModified());
+ // header dates are often junk (e.g. year 138339611)
+ Calendar created = font.getHeader().getCreated();
+ if (created != null && TikaDates.inYearBounds(created.toInstant())) {
+ metadata.set(TikaCoreProperties.CREATED, created);
+ }
+ Calendar modified = font.getHeader().getModified();
+ if (modified != null && TikaDates.inYearBounds(modified.toInstant())) {
+ metadata.set(TikaCoreProperties.MODIFIED, modified);
+ }
metadata.set(Property.externalText(AdobeFontMetricParser.MET_DOC_VERSION),
Float.toString(font.getHeader().getVersion()));
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
index a23684d12b..0ee7e82fbc 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java
@@ -81,6 +81,7 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TIFF;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.utils.TikaDates;
/**
* Uses the <a href="http://www.drewnoakes.com/code/exif/">Metadata Extractor</a> library
@@ -552,13 +553,18 @@ public class ImageMetadataExtractor {
static class ExifHandler implements DirectoryHandler {
// There's a new ExifHandler for each file processed, so this is thread safe
+ // EXIF dates have no zone: read and write in GMT (metadata-extractor >= 2.20 uses the JVM zone)
+ private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
private final SimpleDateFormat dateUnspecifiedTz = getUnspecifiedTzDateFormat();
+ // metadata-extractor turns junk like "2" into year 1
+ private static Date inBounds(Date d) {
+ return d != null && TikaDates.inYearBounds(d.toInstant()) ? d : null;
+ }
+
private SimpleDateFormat getUnspecifiedTzDateFormat() {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US);
- // As of Drew Noakes' metadata-extractor 2.8.1, unspecified
- // timezones are set to TimeZone.getTimeZone("GMT")
- df.setTimeZone(TimeZone.getTimeZone("GMT"));
+ df.setTimeZone(GMT);
return df;
}
@@ -728,18 +734,17 @@ public class ImageMetadataExtractor {
// Date/Time Original overrides value from ExifDirectory.TAG_DATETIME
Date original = null;
if (directory.containsTag(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)) {
- original = directory.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL);
+ original = inBounds(directory.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL, GMT));
// Unless we have GPS time we don't know the time zone so date must be set
// as ISO 8601 datetime without timezone suffix (no Z or +/-)
if (original != null) {
- String datetimeNoTimeZone = dateUnspecifiedTz
- .format(original); // Same time zone as Metadata Extractor uses
+ String datetimeNoTimeZone = dateUnspecifiedTz.format(original);
metadata.set(TikaCoreProperties.CREATED, datetimeNoTimeZone);
metadata.set(TIFF.ORIGINAL_DATE, datetimeNoTimeZone);
}
}
if (directory.containsTag(ExifIFD0Directory.TAG_DATETIME)) {
- Date datetime = directory.getDate(ExifIFD0Directory.TAG_DATETIME);
+ Date datetime = inBounds(directory.getDate(ExifIFD0Directory.TAG_DATETIME, GMT));
if (datetime != null) {
String datetimeNoTimeZone = dateUnspecifiedTz.format(datetime);
metadata.set(TikaCoreProperties.MODIFIED, datetimeNoTimeZone);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageMetadataExtractorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageMetadataExtractorTest.java
index 9f8b0a3ec8..ab93eab488 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageMetadataExtractorTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageMetadataExtractorTest.java
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TIFF;
import org.apache.tika.metadata.TikaCoreProperties;
public class ImageMetadataExtractorTest {
@@ -82,8 +83,8 @@ public class ImageMetadataExtractorTest {
GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
calendar.setTimeInMillis(0);
calendar.set(2000, 0, 1, 0, 0, 0);
- Mockito.when(exif.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL))
- .thenReturn(calendar.getTime()); // UTC timezone as in Metadata Extractor
+ Mockito.when(exif.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL, TimeZone.getTimeZone("GMT")))
+ .thenReturn(calendar.getTime());
Metadata metadata = new Metadata();
new ImageMetadataExtractor.ExifHandler().handle(exif, metadata);
@@ -120,8 +121,8 @@ public class ImageMetadataExtractorTest {
GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
calendar.setTimeInMillis(0);
calendar.set(1999, 0, 1, 0, 0, 0);
- Mockito.when(exif.getDate(ExifIFD0Directory.TAG_DATETIME))
- .thenReturn(calendar.getTime()); // UTC timezone as in Metadata Extractor
+ Mockito.when(exif.getDate(ExifIFD0Directory.TAG_DATETIME, TimeZone.getTimeZone("GMT")))
+ .thenReturn(calendar.getTime());
Metadata metadata = new Metadata();
new ImageMetadataExtractor.ExifHandler().handle(exif, metadata);
@@ -129,11 +130,26 @@ public class ImageMetadataExtractorTest {
"Should try EXIF Date/Time if Original is not set");
}
+ @Test
+ public void testExifHandlerParseDateOutOfBounds() throws MetadataException {
+ ExifSubIFDDirectory exif = Mockito.mock(ExifSubIFDDirectory.class);
+ Mockito.when(exif.containsTag(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)).thenReturn(true);
+ GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+ calendar.setTimeInMillis(0);
+ calendar.set(4, 11, 31, 23, 0, 0);
+ Mockito.when(exif.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL, TimeZone.getTimeZone("GMT")))
+ .thenReturn(calendar.getTime());
+ Metadata metadata = new Metadata();
+ new ImageMetadataExtractor.ExifHandler().handle(exif, metadata);
+ assertNull(metadata.get(TikaCoreProperties.CREATED));
+ assertNull(metadata.get(TIFF.ORIGINAL_DATE));
+ }
+
@Test
public void testExifHandlerParseDateError() throws MetadataException {
ExifIFD0Directory exif = Mockito.mock(ExifIFD0Directory.class);
Mockito.when(exif.containsTag(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)).thenReturn(true);
- Mockito.when(exif.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL)).thenReturn(null);
+ Mockito.when(exif.getDate(ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL, TimeZone.getTimeZone("GMT"))).thenReturn(null);
Metadata metadata = new Metadata();
new ImageMetadataExtractor.ExifHandler().handle(exif, metadata);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/JpegParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/JpegParserTest.java
index 11d6d5a9f0..fc974bebdd 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/JpegParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/JpegParserTest.java
@@ -63,10 +63,9 @@ public class JpegParserTest extends TikaTest {
@BeforeAll
static void init() {
- // metadata-extractor 2.20.0 started making these parsed dates depend on the JVM default time zone;
- // force GMT so the assertions remain deterministic across environments
+ // +14h: EXIF dates must not depend on the JVM zone
originalTimeZone = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
+ TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Kiritimati"));
}
@AfterAll
static void tearDown() {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/pom.xml
index 741af98d80..b707b774cd 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/pom.xml
@@ -28,19 +28,14 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>tika-parser-mail-commons</artifactId>
- <name>Apache Tika mail commons</name>
+ <name>Apache Tika mail commons (deprecated, removed in 4.3.0)</name>
<dependencies>
<dependency>
- <groupId>org.apache.james</groupId>
- <artifactId>apache-mime4j-core</artifactId>
- <version>${mime4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.james</groupId>
- <artifactId>apache-mime4j-dom</artifactId>
- <version>${mime4j.version}</version>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-parser-mail-module</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
<build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailDateParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailDateParser.java
index 471d53100d..2325c31400 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailDateParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailDateParser.java
@@ -64,7 +64,10 @@ import org.apache.tika.utils.StringUtils;
* This code does not spark joy especially given the diffs in behavior between jdk versions.
*
* At some point, we should probably try joda or, heaven forfend, a pile of regexes.
+ *
+ * @deprecated since 4.2.0, removed in 4.3.0; use {@link org.apache.tika.utils.TikaDates}.
*/
+@Deprecated(since = "4.2.0", forRemoval = true)
public class MailDateParser {
//TIKA-1970 Mac Mail's format is GMT+1 so we need to check for hour only
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java
index 057e4ae19c..dac12d1d47 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java
@@ -16,99 +16,9 @@
*/
package org.apache.tika.parser.mailcommons;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.Property;
-
-public class MailUtil {
-
- private final static Pattern EMAIL = Pattern.compile("(?i)([^<@\\s]+@[^@> ]+)");
-
- /**
- * This tries to split a "from" or "to" value into a person field and an email field.
- * This does not handle RFC encoded strings (e.g. "=?iso-8859-1?Q?H=E9roux_Louise?"),
- * you must decode them first.
- *
- * @param string
- * @param personProperty
- * @param emailProperty
- * @param metadata
- */
- public static void setPersonAndEmail(String string, Property personProperty,
- Property emailProperty, Metadata metadata) {
-
- StringBuffer sb = new StringBuffer();
- String email = extractEmail(string, sb);
- String person = clean(sb.toString());
- if (person != null && person.length() > 0) {
- metadata.set(personProperty, person);
- }
- if (email != null && email.length() > 0) {
- metadata.set(emailProperty, email);
- }
-
- }
-
- /**
- * This tries to split a "from" or "to" value into a person field and an email field.
- * This does not handle RFC encoded strings (e.g. "=?iso-8859-1?Q?H=E9roux_Louise?"),
- * you must decode them first.
- *
- * @param string
- * @param personProperty
- * @param emailProperty
- * @param metadata
- */
- public static void addPersonAndEmail(String string, Property personProperty,
- Property emailProperty, Metadata metadata) {
-
- StringBuffer sb = new StringBuffer();
- String email = extractEmail(string, sb);
- String person = clean(sb.toString());
-
- if (person != null && person.length() > 0) {
- metadata.add(personProperty, person);
- }
- if (email != null && email.length() > 0) {
- metadata.add(emailProperty, email);
- }
-
- }
-
- private static String clean(String s) {
- s = s.replaceAll("[<>\"]", " ");
- s = s.trim();
- return s;
- }
-
- private static String extractEmail(String string, StringBuffer sb) {
- Matcher emailMatcher = EMAIL.matcher(string);
- String email = "";
- //TODO: warn if more than one email is found?
- while (emailMatcher.find()) {
- emailMatcher.appendReplacement(sb, "");
- email = emailMatcher.group(1);
- }
- emailMatcher.appendTail(sb);
- return email;
- }
-
- /**
- * If the chunk looks like it contains an email
- *
- * @param chunk to analyse
- * @return {@code true} if chunk looks like it contains an email
- */
- public static boolean containsEmail(String chunk) {
- if (chunk == null) {
- return false;
- }
-
- if (EMAIL.matcher(chunk).find()) {
- return true;
- }
- return false;
- }
+/**
+ * @deprecated since 4.2.0, removed in 4.3.0; use {@link org.apache.tika.parser.mail.MailUtil}.
+ */
+@Deprecated(since = "4.2.0", forRemoval = true)
+public class MailUtil extends org.apache.tika.parser.mail.MailUtil {
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml
index 8a4096a79b..b9991b7d69 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/pom.xml
@@ -31,9 +31,14 @@
<dependencies>
<dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-parser-mail-commons</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.james</groupId>
+ <artifactId>apache-mime4j-core</artifactId>
+ <version>${mime4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.james</groupId>
+ <artifactId>apache-mime4j-dom</artifactId>
+ <version>${mime4j.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
index ed3d156116..136608f940 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
@@ -19,7 +19,6 @@ package org.apache.tika.parser.mail;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
-import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -50,6 +49,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.Metadata;
@@ -60,13 +60,12 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.csv.TextAndCSVParser;
import org.apache.tika.parser.html.JSoupParser;
-import org.apache.tika.parser.mailcommons.MailDateParser;
-import org.apache.tika.parser.mailcommons.MailUtil;
import org.apache.tika.parser.txt.TXTParser;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
* Bridge between mime4j's content handler and the generic Sax content handler
@@ -170,10 +169,13 @@ class MailContentHandler implements ContentHandler {
for (Entry<String, String> param : contentDispositionParameters.entrySet()) {
contentDisposition.append("; ").append(param.getKey()).append("=\"")
.append(param.getValue()).append('"');
+ // RFC 2183: the attached file's file-system dates, not the document's own
if ("creation-date".equalsIgnoreCase(param.getKey())) {
- tryToAddDate(param.getValue(), TikaCoreProperties.CREATED, submd);
+ tryToAddDate(param.getValue(), FileSystem.CREATED, submd);
} else if ("modification-date".equalsIgnoreCase(param.getKey())) {
- tryToAddDate(param.getValue(), TikaCoreProperties.MODIFIED, submd);
+ tryToAddDate(param.getValue(), FileSystem.MODIFIED, submd);
+ } else if ("read-date".equalsIgnoreCase(param.getKey())) {
+ tryToAddDate(param.getValue(), FileSystem.ACCESSED, submd);
}
//do anything with "size"?
}
@@ -199,7 +201,7 @@ class MailContentHandler implements ContentHandler {
}
private void tryToAddDate(String value, Property property, Metadata metadata) {
- Date d = MailDateParser.parseDateLenient(value);
+ String d = TikaDates.toMetadataString(value);
if (d != null) {
metadata.set(property, d);
}
@@ -374,16 +376,7 @@ class MailContentHandler implements ContentHandler {
metadata.add(Message.RAW_HEADER, parsedField.getName(), field.getBody());
}
} else if (fieldname.equalsIgnoreCase("Date")) {
- String dateBody = parsedField.getBody();
- Date date = null;
- try {
- date = MailDateParser.parseDateLenient(dateBody);
- metadata.set(TikaCoreProperties.CREATED, date);
- } catch (SecurityException e) {
- throw e;
- } catch (Exception e) {
- //swallow
- }
+ tryToAddDate(parsedField.getBody(), TikaCoreProperties.CREATED, metadata);
} else {
metadata.add(Message.RAW_HEADER, parsedField.getName(), field.getBody());
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailUtil.java
similarity index 98%
copy from tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java
copy to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailUtil.java
index 057e4ae19c..79c610eeef 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/main/java/org/apache/tika/parser/mailcommons/MailUtil.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/MailUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.mailcommons;
+package org.apache.tika.parser.mail;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
index da909baac9..21d4649352 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
@@ -16,13 +16,10 @@
*/
package org.apache.tika.parser.mbox;
-import static org.apache.tika.parser.mailcommons.MailDateParser.parseDateLenient;
-
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Collections;
-import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Locale;
@@ -51,9 +48,10 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.mailcommons.MailUtil;
+import org.apache.tika.parser.mail.MailUtil;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
* Mbox (mailbox) parser. This version extracts each mail from Mbox and uses the
@@ -211,15 +209,9 @@ public class MboxParser implements Parser {
} else if (headerTag.equalsIgnoreCase("Subject")) {
metadata.add(TikaCoreProperties.SUBJECT, headerContent);
} else if (headerTag.equalsIgnoreCase("Date")) {
- try {
- Date date = parseDateLenient(headerContent);
- if (date != null) {
- metadata.set(TikaCoreProperties.CREATED, date);
- }
- } catch (SecurityException e) {
- throw e;
- } catch (Exception e) {
- // ignoring date because format was not understood
+ String date = TikaDates.toMetadataString(headerContent);
+ if (date != null) {
+ metadata.set(TikaCoreProperties.CREATED, date);
}
} else if (headerTag.equalsIgnoreCase("Message-Id")) {
metadata.set(TikaCoreProperties.IDENTIFIER, headerContent);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/test/java/org/apache/tika/parser/mailcommons/MailUtilTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/MailUtilTest.java
similarity index 97%
rename from tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/test/java/org/apache/tika/parser/mailcommons/MailUtilTest.java
rename to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/MailUtilTest.java
index 8087c26904..df0e3fc5c5 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-commons/src/test/java/org/apache/tika/parser/mailcommons/MailUtilTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/MailUtilTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.mailcommons;
+package org.apache.tika.parser.mail;
import static org.junit.jupiter.api.Assertions.assertEquals;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
index 17412fed55..8e4134b67a 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
@@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
@@ -46,6 +47,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.ContainerExtractor;
import org.apache.tika.extractor.ParserContainerExtractor;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.Metadata;
@@ -60,6 +62,45 @@ import org.apache.tika.sax.XHTMLContentHandler;
public class RFC822ParserTest extends TikaTest {
+ @Test
+ public void testDateHeaderRepresentation() throws Exception {
+ assertEquals("2000-12-01T08:39:07", createdFor("Fri, 01 Dec 2000 08:39:07"));
+ assertEquals("2000-12-01T07:39:07Z", createdFor("Fri, 01 Dec 2000 08:39:07 +0100"));
+ assertEquals("1992-04-10T11:00:33Z", createdFor("Fri Apr 10 04:00:33 PDT 1992")); // ctime
+ assertEquals(null, createdFor("not a date"));
+ }
+
+ @Test
+ public void testAttachmentDatesAreFileSystemDates() throws Exception {
+ String eml = "From: a@example.com\r\nSubject: s\r\nMIME-Version: 1.0\r\n"
+ + "Content-Type: multipart/mixed; boundary=\"b\"\r\n\r\n"
+ + "--b\r\nContent-Type: text/plain\r\n\r\nbody\r\n"
+ + "--b\r\nContent-Type: text/plain\r\nContent-Disposition: attachment; filename=\"a.txt\";"
+ + " creation-date=\"Tue, 21 Oct 2003 10:11:12 +0200\";"
+ + " modification-date=\"Wed, 22 Oct 2003 10:11:12 +0200\";"
+ + " read-date=\"Thu, 23 Oct 2003 10:11:12 +0200\"\r\n\r\nattached\r\n--b--\r\n";
+ List<Metadata> list;
+ try (TikaInputStream tis = TikaInputStream.get(eml.getBytes(US_ASCII))) {
+ list = getRecursiveMetadata(tis, AUTO_DETECT_PARSER, new Metadata(), new ParseContext(), false);
+ }
+ Metadata att = list.stream().filter(m -> "a.txt".equals(m.get(TikaCoreProperties.RESOURCE_NAME_KEY)))
+ .findFirst().orElseThrow();
+ assertEquals("2003-10-21T08:11:12Z", att.get(FileSystem.CREATED));
+ assertEquals("2003-10-22T08:11:12Z", att.get(FileSystem.MODIFIED));
+ assertEquals("2003-10-23T08:11:12Z", att.get(FileSystem.ACCESSED));
+ assertNull(att.get(TikaCoreProperties.CREATED));
+ assertNull(att.get(TikaCoreProperties.MODIFIED));
+ }
+
+ private String createdFor(String dateHeader) throws Exception {
+ String eml = "From: a@example.com\r\nDate: " + dateHeader + "\r\nSubject: s\r\n\r\nbody\r\n";
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(eml.getBytes(US_ASCII))) {
+ new RFC822Parser().parse(tis, new DefaultHandler(), metadata, new ParseContext());
+ }
+ return metadata.get(TikaCoreProperties.CREATED);
+ }
+
//legacy RFC822 behavior...extract every alternative part
private static Parser EXTRACT_ALL_ALTERNATIVES_PARSER;
@@ -592,9 +633,10 @@ public class RFC822ParserTest extends TikaTest {
assertEquals("/Test TxtA.txt", metadataList.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
assertEquals(TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString(),
metadataList.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
- //make sure we extracted creation and modified dates
- assertTrue(metadataList.get(1).get(TikaCoreProperties.CREATED).startsWith("2022-11-"));
- assertTrue(metadataList.get(1).get(TikaCoreProperties.MODIFIED).startsWith("2022-11-"));
+ // Content-Disposition dates are the attached file's, not the document's
+ assertTrue(metadataList.get(1).get(FileSystem.CREATED).startsWith("2022-11-"));
+ assertTrue(metadataList.get(1).get(FileSystem.MODIFIED).startsWith("2022-11-"));
+ assertNull(metadataList.get(1).get(TikaCoreProperties.CREATED));
assertContains("This is Test TXTB File for parser",
@@ -605,9 +647,8 @@ public class RFC822ParserTest extends TikaTest {
metadataList.get(2).get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
assertEquals(TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString(),
metadataList.get(2).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
- //make sure we extracted creation and modified dates
- assertTrue(metadataList.get(2).get(TikaCoreProperties.CREATED).startsWith("2022-11-"));
- assertTrue(metadataList.get(2).get(TikaCoreProperties.MODIFIED).startsWith("2022-11-"));
+ assertTrue(metadataList.get(2).get(FileSystem.CREATED).startsWith("2022-11-"));
+ assertTrue(metadataList.get(2).get(FileSystem.MODIFIED).startsWith("2022-11-"));
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
index e42bab1ce7..b91f771198 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
@@ -70,11 +70,16 @@
<artifactId>tika-parser-xml-module</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- needed for msg files -->
+ <!-- needed for msg files (transport headers) -->
<dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>tika-parser-mail-commons</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.james</groupId>
+ <artifactId>apache-mime4j-core</artifactId>
+ <version>${mime4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.james</groupId>
+ <artifactId>apache-mime4j-dom</artifactId>
+ <version>${mime4j.version}</version>
</dependency>
<dependency>
<groupId>com.pff</groupId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
index 13a42c5e13..efecc69494 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
@@ -31,7 +31,6 @@ import java.nio.charset.UnsupportedCharsetException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -83,7 +82,6 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.html.JSoupParser;
-import org.apache.tika.parser.mailcommons.MailDateParser;
import org.apache.tika.parser.microsoft.msg.ExtendedMetadataExtractor;
import org.apache.tika.parser.microsoft.rtf.RTFParser;
import org.apache.tika.parser.microsoft.rtf.jflex.RTFHtmlDecapsulator;
@@ -91,6 +89,7 @@ import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
@@ -558,17 +557,10 @@ public class OutlookExtractor extends AbstractPOIFSExtractor {
if (headerKey.toLowerCase(Locale.ROOT).startsWith("date:")) {
String date = headerKey.substring(headerKey.indexOf(':') + 1).trim();
- // See if we can parse it as a normal mail date
- try {
- Date d = MailDateParser.parseDateLenient(date);
+ String d = TikaDates.toMetadataString(date);
+ if (d != null) {
metadata.set(TikaCoreProperties.CREATED, d);
metadata.set(TikaCoreProperties.MODIFIED, d);
- } catch (SecurityException e) {
- throw e;
- } catch (Exception e) {
- // Store it as-is, and hope for the best...
- metadata.set(TikaCoreProperties.CREATED, date);
- metadata.set(TikaCoreProperties.MODIFIED, date);
}
break;
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java
index eeeabdc7e7..542420c9b0 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/SummaryExtractor.java
@@ -47,6 +47,7 @@ import org.apache.tika.metadata.PagedText;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
* Extractor for Common OLE2 (HPSF) metadata
@@ -258,7 +259,8 @@ public class SummaryExtractor {
}
private void set(Property property, Date value) {
- if (value != null) {
+ // FILETIMEs are often junk (e.g. year 31135)
+ if (value != null && TikaDates.inYearBounds(value.toInstant())) {
metadata.set(property, value);
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
index 4f591ae4c6..e9aa3f001a 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
@@ -25,7 +25,7 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
-import org.apache.tika.utils.DateUtils;
+import org.apache.tika.utils.TikaDates;
/**
* This class is intended to handle anything that might contain IBodyElements:
@@ -174,7 +174,6 @@ public class OOXMLWordAndPowerPointTextHandler extends DefaultHandler {
private final StringBuilder instrTextBuffer = new StringBuilder();
private EditType editType =
EditType.NONE;
- private DateUtils dateUtils = new DateUtils();
private boolean hiddenSlide = false;
private boolean hasAnimations = false;
@@ -481,7 +480,8 @@ public class OOXMLWordAndPowerPointTextHandler extends DefaultHandler {
String editDateString = atts.getValue(W_NS, "date");
Date editDate = null;
if (editDateString != null) {
- editDate = dateUtils.tryToParse(editDateString);
+ editDate = TikaDates.parse(editDateString).filter(TikaDates.ParsedDate::isFullPrecision)
+ .map(d -> Date.from(d.toInstant())).orElse(null);
}
bodyContentsHandler.startEditedSection(editAuthor, editDate, editType);
this.editType = editType;
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
index 9d8ac8c6d1..1288c1ff51 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
@@ -28,13 +28,15 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
-import java.util.Calendar;
+import java.time.DateTimeException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Locale;
import java.util.Map;
import java.util.Stack;
-import java.util.TimeZone;
import org.apache.commons.io.IOUtils;
import org.xml.sax.ContentHandler;
@@ -53,6 +55,7 @@ import org.apache.tika.metadata.RTFMetadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.utils.CharsetUtils;
+import org.apache.tika.utils.TikaDates;
/* Tokenizes and performs a "shallow" parse of the RTF
* document, just enough to properly decode the text.
@@ -321,8 +324,11 @@ final class TextExtractor {
// Used to process the sub-groups inside the upr
// group:
private int uprState = -1;
- // Used when extracting CREATION date:
- private int year, month, day, hour, minute;
+ // \\creatim fields, reset per group
+ private int year = -1, month = -1, day = -1, hour = -1, minute = -1;
+ private static final DateTimeFormatter ZONELESS =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT);
+ private static final DateTimeFormatter DATE_ONLY = DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ROOT);
//This keeps track of the following elements as they are
//written to the handler: p, li, ol, ul
@@ -1162,6 +1168,7 @@ final class TextExtractor {
nextMetaData = OfficeOpenXMLExtended.TEMPLATE;
} else if (equals("creatim")) {
nextMetaData = TikaCoreProperties.CREATED;
+ year = month = day = hour = minute = -1;
}
}
@@ -1478,14 +1485,30 @@ final class TextExtractor {
in.unread(b2);
}
+ // zone-less; missing or invalid fields -> no date
+ private String creationDate() {
+ if (year < TikaDates.MIN_YEAR || year > TikaDates.MAX_YEAR) {
+ return null;
+ }
+ try {
+ if (hour < 0) {
+ return DATE_ONLY.format(LocalDate.of(year, month, day));
+ }
+ return ZONELESS.format(LocalDateTime.of(year, month, day, hour, Math.max(minute, 0)));
+ } catch (DateTimeException e) {
+ return null;
+ }
+ }
+
// Pop current GroupState
private void processGroupEnd() throws IOException, SAXException, TikaException {
if (inHeader) {
if (nextMetaData != null) {
if (nextMetaData == TikaCoreProperties.CREATED) {
- Calendar cal = Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT);
- cal.set(year, month - 1, day, hour, minute, 0);
- metadata.set(nextMetaData, cal.getTime());
+ String created = creationDate();
+ if (created != null) {
+ metadata.set(nextMetaData, created);
+ }
} else if (nextMetaData.isMultiValuePermitted()) {
metadata.add(nextMetaData, pendingBuffer.toString());
} else {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
index 0e23f73529..e9c5950027 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
@@ -18,16 +18,19 @@ package org.apache.tika.parser.microsoft.rtf;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.junit.jupiter.api.Test;
+import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.Tika;
import org.apache.tika.TikaTest;
@@ -423,9 +426,28 @@ public class RTFParserTest extends TikaTest {
assertEquals("1", xml.metadata.get(Office.PAGE_COUNT));
assertEquals("70", xml.metadata.get(Office.WORD_COUNT));
assertEquals("401", xml.metadata.get(Office.CHARACTER_COUNT));
- //RTFParser's legacy behavior is to apply local timezone to dates/times.
- //This needs to be flexible enough to pass in various time-zones TIKA-4043
- assertTrue(xml.metadata.get(TikaCoreProperties.CREATED).startsWith("2010-10-"));
+ assertEquals("2010-10-13T02:55:00", xml.metadata.get(TikaCoreProperties.CREATED));
+ }
+
+ @Test
+ public void testCreationDateFields() throws Exception {
+ assertEquals("2012-03-04T05:06:00", created("{\\creatim\\yr2012\\mo3\\dy4\\hr5\\min6}"));
+ assertEquals("2012-03-04", created("{\\creatim\\yr2012\\mo3\\dy4}")); // no time: date-only
+ assertEquals("2012-03-04T05:00:00", created("{\\creatim\\yr2012\\mo3\\dy4\\hr5}"));
+ assertNull(created("{\\creatim}")); // was a negative year
+ assertNull(created("{\\creatim\\yr0\\mo0\\dy0\\hr0\\min0}"));
+ assertNull(created("{\\creatim\\yr2012\\mo13\\dy40}")); // was rolled over into 2013
+ // no leak from an earlier date group
+ assertNull(created("{\\printim\\yr2011\\mo1\\dy1}{\\creatim}"));
+ }
+
+ private String created(String infoGroups) throws Exception {
+ String rtf = "{\\rtf1\\ansi{\\info" + infoGroups + "}hello}";
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(rtf.getBytes(StandardCharsets.US_ASCII))) {
+ new RTFParser().parse(tis, new DefaultHandler(), metadata, new ParseContext());
+ }
+ return metadata.get(TikaCoreProperties.CREATED);
}
// TIKA-1192
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/DateElementHandler.java
similarity index 50%
copy from tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java
copy to tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/DateElementHandler.java
index dc161cbfa8..bfd2417e49 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/DateElementHandler.java
@@ -14,75 +14,55 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.parser.xml;
+package org.apache.tika.parser.odf;
import org.xml.sax.Attributes;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
+import org.apache.tika.utils.TikaDates;
-/**
- * This adds Metadata entries with a specified name for
- * the textual content of a node (if present), and
- * all attribute values passed through the matcher
- * (but not their names).
- *
- * @deprecated Use the {@link AttributeMetadataHandler} and
- * {@link ElementMetadataHandler} classes instead
- */
-@Deprecated
-public class MetadataHandler extends DefaultHandler {
+/** ODF meta.xml date element; invalid values are dropped. */
+class DateElementHandler extends DefaultHandler {
+ private final String namespace;
+ private final String localName;
private final Metadata metadata;
-
private final Property property;
- private final String name;
-
private final StringBuilder buffer = new StringBuilder();
+ private boolean inElement;
- /**
- * String-keyed constructor: {@code name} must be a fixed, compile-time-known key
- * name (never document-derived), so it is minted as a curated, registered Property,
- * like {@link Property#externalText}.
- */
- public MetadataHandler(Metadata metadata, String name) {
- this(metadata, Property.externalText(name));
- }
-
- public MetadataHandler(Metadata metadata, Property property) {
+ DateElementHandler(String namespace, String localName, Metadata metadata, Property property) {
+ this.namespace = namespace;
+ this.localName = localName;
this.metadata = metadata;
this.property = property;
- this.name = property.getName();
- }
-
- public void addMetadata(String value) {
- if (value.length() > 0) {
- String previous = metadata.get(name);
- if (previous != null && previous.length() > 0) {
- value = previous + ", " + value;
- }
- metadata.set(property, value);
- }
}
@Override
- public void endElement(String uri, String localName, String name) {
- addMetadata(buffer.toString());
- buffer.setLength(0);
+ public void startElement(String uri, String localName, String qName, Attributes attributes) {
+ if (namespace.equals(uri) && this.localName.equals(localName)) {
+ inElement = true;
+ buffer.setLength(0);
+ }
}
@Override
- public void startElement(String uri, String localName, String name, Attributes attributes) {
- for (int i = 0; i < attributes.getLength(); i++) {
- addMetadata(attributes.getValue(i));
+ public void characters(char[] ch, int start, int length) {
+ if (inElement) {
+ buffer.append(ch, start, length);
}
}
-
@Override
- public void characters(char[] ch, int start, int length) {
- buffer.append(ch, start, length);
+ public void endElement(String uri, String localName, String qName) {
+ if (inElement && namespace.equals(uri) && this.localName.equals(localName)) {
+ inElement = false;
+ String value = TikaDates.toMetadataString(buffer.toString());
+ if (value != null) {
+ metadata.set(property, value);
+ }
+ }
}
-
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java
index 4fddc6d385..3842d6958d 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentMetaParser.java
@@ -134,10 +134,11 @@ public class OpenDocumentMetaParser extends XMLParser {
getDublinCoreHandler(md, TikaCoreProperties.RIGHTS, "rights"));
ch = getVersion(ch, md);
// Process the OO Meta Attributes
- ch = getMeta(ch, md, TikaCoreProperties.CREATED, "creation-date");
+ ch = new TeeContentHandler(ch,
+ new DateElementHandler(META_NS, "creation-date", md, TikaCoreProperties.CREATED));
// ODF uses dc:date for modified
ch = new TeeContentHandler(ch,
- new ElementMetadataHandler(DublinCore.NAMESPACE_URI_DC, "date", md,
+ new DateElementHandler(DublinCore.NAMESPACE_URI_DC, "date", md,
TikaCoreProperties.MODIFIED));
// ODF uses dc:subject for description
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
index bec85a4c2e..bec3adbc96 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
@@ -85,6 +85,41 @@ public class ODFParserTest extends TikaTest {
}
}
+ @Test
+ public void testContainerDateNotJoined() throws Exception {
+ try (TikaInputStream tis = getResourceAsStream("/test-documents/testOpenOffice2.odt")) {
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.CREATED, "2020-01-01T00:00:00.123456789Z");
+ new OpenDocumentParser().parse(tis, new BodyContentHandler(), metadata, new ParseContext());
+ assertArrayEquals(new String[]{"2007-09-14T11:06:08"},
+ metadata.getValues(TikaCoreProperties.CREATED));
+ }
+ }
+
+ @Test
+ public void testMetaDates() throws Exception {
+ assertArrayEquals(new String[]{"2011-03-04T10:22:33", "2011-03-05T01:02:03Z"},
+ metaDates("2011-03-04T10:22:33.123456789", "2011-03-05T02:02:03+01:00"));
+ assertArrayEquals(new String[]{null, null},
+ metaDates("0101-01-01T00:00:00+00:00", "2018--0-8-T23: 1:4:+01:00"));
+ assertArrayEquals(new String[]{"2012-06-01", null}, metaDates("2012-06-01", "May 1917"));
+ }
+
+ private String[] metaDates(String created, String modified) throws Exception {
+ String xml = "<?xml version='1.0' encoding='UTF-8'?>"
+ + "<office:document-meta xmlns:office='urn:oasis:names:tc:opendocument:xmlns:office:1.0'"
+ + " xmlns:meta='urn:oasis:names:tc:opendocument:xmlns:meta:1.0'"
+ + " xmlns:dc='http://purl.org/dc/elements/1.1/'><office:meta>"
+ + "<meta:creation-date>" + created + "</meta:creation-date>"
+ + "<dc:date>" + modified + "</dc:date>"
+ + "</office:meta></office:document-meta>";
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(xml.getBytes(StandardCharsets.UTF_8))) {
+ new OpenDocumentMetaParser().parse(tis, new DefaultHandler(), metadata, new ParseContext());
+ }
+ return new String[]{metadata.get(TikaCoreProperties.CREATED), metadata.get(TikaCoreProperties.MODIFIED)};
+ }
+
@Test
public void testOO2() throws Exception {
for (Parser parser : getParsers()) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/AbstractArchiveParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/AbstractArchiveParser.java
index 7fba796b91..e43047e2bf 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/AbstractArchiveParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/AbstractArchiveParser.java
@@ -17,19 +17,27 @@
package org.apache.tika.parser.pkg;
import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Date;
+import java.util.Locale;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.AbstractEncodingDetectorParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.XHTMLContentHandler;
+import org.apache.tika.utils.TikaDates;
/**
* Abstract base class for archive parsers that provides common functionality
@@ -37,6 +45,9 @@ import org.apache.tika.sax.XHTMLContentHandler;
*/
public abstract class AbstractArchiveParser extends AbstractEncodingDetectorParser {
+ private static final DateTimeFormatter ZONELESS =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT);
+
public AbstractArchiveParser() {
super();
}
@@ -49,8 +60,8 @@ public abstract class AbstractArchiveParser extends AbstractEncodingDetectorPars
* Handles metadata for an archive entry and writes appropriate XHTML elements.
*
* @param name the entry name
- * @param createAt creation date (may be null)
- * @param modifiedAt modification date (may be null)
+ * @param createAt creation instant (may be null), stored as {@link FileSystem#CREATED}
+ * @param modifiedAt modification instant (may be null), stored as {@link FileSystem#MODIFIED}
* @param size entry size (may be null)
* @param xhtml the XHTML content handler
* @param context the parse context
@@ -61,12 +72,9 @@ public abstract class AbstractArchiveParser extends AbstractEncodingDetectorPars
ParseContext context)
throws SAXException, IOException, TikaException {
Metadata entrydata = Metadata.newInstance(context);
- if (createAt != null) {
- entrydata.set(TikaCoreProperties.CREATED, createAt);
- }
- if (modifiedAt != null) {
- entrydata.set(TikaCoreProperties.MODIFIED, modifiedAt);
- }
+ // entry times are file-system times, never the embedded document's own dates
+ setInstant(entrydata, FileSystem.CREATED, createAt);
+ setInstant(entrydata, FileSystem.MODIFIED, modifiedAt);
if (size != null) {
entrydata.set(HttpHeaders.CONTENT_LENGTH, Long.toString(size));
}
@@ -82,4 +90,29 @@ public abstract class AbstractArchiveParser extends AbstractEncodingDetectorPars
}
return entrydata;
}
+
+ /** An archive-header instant; junk years (e.g. a garbage FILETIME) are not stored. */
+ static void setInstant(Metadata metadata, Property property, Date date) {
+ if (date != null && TikaDates.inYearBounds(date.toInstant())) {
+ metadata.set(property, date);
+ }
+ }
+
+ /**
+ * DOS-style local time the library resolved in the JVM zone: undo that, store zone-less.
+ * Not exact inside a DST spring-forward gap (02:30 comes back as 03:30); only the raw
+ * header field can fix that, which zip (via {@link ZipParser#readDosTime}) reads and RAR/ARJ don't expose.
+ */
+ static void setLocalTime(Metadata metadata, Property property, Date resolvedInDefaultZone) {
+ if (resolvedInDefaultZone != null) {
+ setLocalTime(metadata, property,
+ LocalDateTime.ofInstant(resolvedInDefaultZone.toInstant(), ZoneId.systemDefault()));
+ }
+ }
+
+ static void setLocalTime(Metadata metadata, Property property, LocalDateTime local) {
+ if (local != null && TikaDates.inYearBounds(local.toInstant(ZoneOffset.UTC))) {
+ metadata.set(property, ZONELESS.format(local));
+ }
+ }
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java
index 68d3198479..91af938978 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java
@@ -36,8 +36,10 @@ import static org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.Channels;
+import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -70,6 +72,7 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.FilenameUtils;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -78,6 +81,7 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
* Parser for various compression formats.
@@ -329,13 +333,18 @@ public class CompressorParser implements Parser {
if (gzipParameters == null) {
return false;
}
+ // MTIME 0 means no timestamp (RFC 1952)
+ Instant mtime = gzipParameters.getModificationInstant();
+ if (mtime != null && mtime.getEpochSecond() != 0 && TikaDates.inYearBounds(mtime)) {
+ metadata.set(FileSystem.MODIFIED, Date.from(mtime));
+ }
String name = gzipParameters.getFileName();
if (!StringUtils.isBlank(name)) {
metadata.set(TikaCoreProperties.INTERNAL_PATH, name);
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, name);
return true;
}
- //TODO: modification, OS, comment
+ //TODO: OS, comment
return false;
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
index f0441e5e4b..5ca67fb341 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
@@ -31,6 +31,7 @@ import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
+import org.apache.commons.compress.archivers.arj.ArjArchiveEntry;
import org.apache.commons.compress.archivers.arj.ArjArchiveInputStream;
import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream;
import org.apache.commons.compress.archivers.dump.DumpArchiveInputStream;
@@ -45,6 +46,7 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -171,9 +173,14 @@ public class PackageParser extends AbstractArchiveParser {
String name = entry.getName();
if (archive.canReadEntryData(entry)) {
+ // non-Unix arj stores DOS local time; the rest are epoch instants
+ boolean localTime = entry instanceof ArjArchiveEntry && !((ArjArchiveEntry) entry).isHostOsUnix();
Metadata entrydata = handleEntryMetadata(
- name, null, entry.getLastModifiedDate(), entry.getSize(),
+ name, null, localTime ? null : entry.getLastModifiedDate(), entry.getSize(),
xhtml, context);
+ if (localTime) {
+ setLocalTime(entrydata, FileSystem.MODIFIED, entry.getLastModifiedDate());
+ }
if (extractor.shouldParseEmbedded(entrydata, context)) {
TemporaryResources tmp = new TemporaryResources();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
index 2b25f38418..667c983612 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/RarParser.java
@@ -34,6 +34,7 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -85,8 +86,11 @@ public class RarParser implements Parser {
while (header != null && !Thread.currentThread().isInterrupted()) {
if (!header.isDirectory()) {
Metadata entrydata = AbstractArchiveParser.handleEntryMetadata(
- header.getFileName(), header.getCTime(), header.getMTime(),
- header.getFullUnpackSize(), xhtml, context);
+ header.getFileName(), null, null, header.getFullUnpackSize(), xhtml, context);
+ // RAR4 times are DOS local time; junrar resolves them in the JVM default zone
+ AbstractArchiveParser.setLocalTime(entrydata, FileSystem.CREATED, header.getCTime());
+ AbstractArchiveParser.setLocalTime(entrydata, FileSystem.MODIFIED, header.getMTime());
+ AbstractArchiveParser.setLocalTime(entrydata, FileSystem.ACCESSED, header.getATime());
try (TikaInputStream rarTis = TikaInputStream.get(rar.getInputStream(header))) {
if (extractor.shouldParseEmbedded(entrydata, context)) {
extractor.parseEmbedded(rarTis, handler, entrydata, context, true);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
index a57c8fbca4..9c27dd9c9d 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
@@ -41,6 +41,7 @@ import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -171,6 +172,9 @@ public class SevenZParser extends AbstractArchiveParser {
entry.getSize(),
xhtml,
context);
+ if (entry.getHasAccessDate()) {
+ setInstant(entrydata, FileSystem.ACCESSED, entry.getAccessDate());
+ }
if (extractor.shouldParseEmbedded(entrydata, context)) {
TemporaryResources tmp = new TemporaryResources();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
index d0e012ba11..28e3830c15 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
@@ -20,11 +20,16 @@ import static org.apache.tika.detect.zip.PackageConstants.JAR;
import static org.apache.tika.detect.zip.PackageConstants.ZIP;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SeekableByteChannel;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
-import java.nio.file.attribute.FileTime;
+import java.time.DateTimeException;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -35,6 +40,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.zip.UnsupportedZipFeatureException;
import org.apache.commons.compress.archivers.zip.UnsupportedZipFeatureException.Feature;
+import org.apache.commons.compress.archivers.zip.X000A_NTFS;
+import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipFile;
@@ -55,6 +62,7 @@ import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -98,6 +106,8 @@ import org.apache.tika.zip.utils.ZipFileHelper;
@TikaComponent()
public class ZipParser extends AbstractArchiveParser {
+ private static final int LOCAL_FILE_HEADER_SIG = 0x04034b50;
+
/**
* Set of media types that are specializations of ZIP (e.g., Office documents, EPUB, APK).
* Used to avoid overwriting more specific media types with generic "application/zip".
@@ -278,7 +288,8 @@ public class ZipParser extends AbstractArchiveParser {
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context);
xhtml.startDocument();
- try {
+ // a channel of our own for raw header reads, so the ZipFile's position is never disturbed
+ try (SeekableByteChannel headers = openHeaderChannel(tis)) {
Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
while (entries.hasMoreElements()) {
ZipArchiveEntry entry = entries.nextElement();
@@ -286,7 +297,7 @@ public class ZipParser extends AbstractArchiveParser {
centralDirectoryEntries.add(entry.getName());
}
if (!entry.isDirectory()) {
- parseZipFileEntry(zipFile, entry, extractor, metadata, xhtml, context, config);
+ parseZipFileEntry(zipFile, headers, entry, extractor, metadata, xhtml, context, config);
}
}
} finally {
@@ -464,7 +475,7 @@ public class ZipParser extends AbstractArchiveParser {
}
}
- private void parseZipFileEntry(ZipFile zipFile, ZipArchiveEntry entry,
+ private void parseZipFileEntry(ZipFile zipFile, SeekableByteChannel headers, ZipArchiveEntry entry,
EmbeddedDocumentExtractor extractor, Metadata parentMetadata,
XHTMLContentHandler xhtml, ParseContext context,
ZipParserConfig config)
@@ -492,7 +503,7 @@ public class ZipParser extends AbstractArchiveParser {
return;
}
- Metadata entryMetadata = buildEntryMetadata(entry, name, context);
+ Metadata entryMetadata = buildEntryMetadata(entry, name, context, readDosTime(headers, entry));
writeEntryXhtml(name, xhtml);
@@ -615,8 +626,88 @@ public class ZipParser extends AbstractArchiveParser {
}
}
+ /** 0x5455/0x000A extra fields are UTC; otherwise only the DOS field: local time, stored zone-less. */
+ static void setEntryTimes(ZipArchiveEntry entry, Metadata md) {
+ setEntryTimes(entry, md, null);
+ }
+
+ private static SeekableByteChannel openHeaderChannel(TikaInputStream tis) {
+ try {
+ return tis.getSeekableByteChannel();
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ /** @param rawDosTime the local header's DOS time, when it could be read; see {@link #readDosTime} */
+ static void setEntryTimes(ZipArchiveEntry entry, Metadata md, LocalDateTime rawDosTime) {
+ X5455_ExtendedTimestamp ts = entry.getExtraField(X5455_ExtendedTimestamp.HEADER_ID)
+ instanceof X5455_ExtendedTimestamp x ? x : null;
+ X000A_NTFS ntfs = entry.getExtraField(X000A_NTFS.HEADER_ID) instanceof X000A_NTFS n ? n : null;
+ Date modified = ts != null && ts.isBit0_modifyTimePresent() ? ts.getModifyJavaTime()
+ : ntfs != null ? ntfs.getModifyJavaTime() : null;
+ if (modified != null) {
+ AbstractArchiveParser.setInstant(md, FileSystem.MODIFIED, modified);
+ } else if (rawDosTime != null) {
+ AbstractArchiveParser.setLocalTime(md, FileSystem.MODIFIED, rawDosTime);
+ } else if (entry.getTime() != -1) {
+ AbstractArchiveParser.setLocalTime(md, FileSystem.MODIFIED, new Date(entry.getTime()));
+ }
+ Date created = ts != null && ts.isBit2_createTimePresent() ? ts.getCreateJavaTime()
+ : ntfs != null ? ntfs.getCreateJavaTime() : null;
+ AbstractArchiveParser.setInstant(md, FileSystem.CREATED, created);
+ Date accessed = ts != null && ts.isBit1_accessTimePresent() ? ts.getAccessJavaTime()
+ : ntfs != null ? ntfs.getAccessJavaTime() : null;
+ AbstractArchiveParser.setInstant(md, FileSystem.ACCESSED, accessed);
+ }
+
+ /**
+ * The DOS wall clock straight from the entry's local file header. commons-compress resolves
+ * it in the JVM zone, which moves a time inside a DST spring-forward gap by an hour.
+ *
+ * @return null if the header can't be read or holds no valid date
+ */
+ static LocalDateTime readDosTime(SeekableByteChannel channel, ZipArchiveEntry entry) {
+ long offset = entry.getLocalHeaderOffset();
+ if (channel == null || offset < 0) {
+ return null;
+ }
+ ByteBuffer header = ByteBuffer.allocate(14).order(ByteOrder.LITTLE_ENDIAN);
+ try {
+ if (offset > channel.size() - header.capacity()) {
+ return null;
+ }
+ channel.position(offset);
+ while (header.hasRemaining()) {
+ if (channel.read(header) < 0) {
+ return null;
+ }
+ }
+ } catch (IOException e) {
+ return null;
+ }
+ if (header.getInt(0) != LOCAL_FILE_HEADER_SIG) {
+ return null;
+ }
+ // time at 10, date at 12: sec/2 min hour | day month year-1980
+ int time = header.getShort(10) & 0xffff;
+ int date = header.getShort(12) & 0xffff;
+ try {
+ return LocalDateTime.of(((date >> 9) & 0x7f) + 1980, (date >> 5) & 0x0f, date & 0x1f,
+ (time >> 11) & 0x1f, (time >> 5) & 0x3f, (time & 0x1f) * 2);
+ } catch (DateTimeException e) {
+ return null;
+ }
+ }
+
private Metadata buildEntryMetadata(ZipArchiveEntry entry, String name, ParseContext context)
throws IOException, TikaException, SAXException {
+ return buildEntryMetadata(entry, name, context, null);
+ }
+
+ private Metadata buildEntryMetadata(ZipArchiveEntry entry, String name, ParseContext context,
+ LocalDateTime rawDosTime)
+ throws IOException, TikaException, SAXException {
Metadata entryMetadata = Metadata.newInstance(context);
if (name != null && name.length() > 0) {
@@ -625,14 +716,7 @@ public class ZipParser extends AbstractArchiveParser {
entryMetadata.set(TikaCoreProperties.INTERNAL_PATH, name);
}
- FileTime creationTime = entry.getCreationTime();
- if (creationTime != null) {
- entryMetadata.set(TikaCoreProperties.CREATED, creationTime.toInstant().toString());
- }
- FileTime modifiedTime = entry.getLastModifiedTime();
- if (modifiedTime != null) {
- entryMetadata.set(TikaCoreProperties.MODIFIED, modifiedTime.toInstant().toString());
- }
+ setEntryTimes(entry, entryMetadata, rawDosTime);
long size = entry.getSize();
if (size >= 0) {
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/AbstractPkgTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/AbstractPkgTest.java
index 073940878f..da84193768 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/AbstractPkgTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/AbstractPkgTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.tika.parser.pkg;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -28,6 +30,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.TikaTest;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -110,8 +113,10 @@ public abstract class AbstractPkgTest extends TikaTest {
ParseContext context) throws IOException, SAXException, TikaException {
filenames.add(metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY));
mediatypes.add(metadata.get(HttpHeaders.CONTENT_TYPE));
- createdAts.add(metadata.get(TikaCoreProperties.CREATED));
- modifiedAts.add(metadata.get(TikaCoreProperties.MODIFIED));
+ assertNull(metadata.get(TikaCoreProperties.CREATED));
+ assertNull(metadata.get(TikaCoreProperties.MODIFIED));
+ createdAts.add(metadata.get(FileSystem.CREATED));
+ modifiedAts.add(metadata.get(FileSystem.MODIFIED));
lastSeenStart = new byte[32];
stream.read(lastSeenStart);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ArchiveTimeBoundsTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ArchiveTimeBoundsTest.java
new file mode 100644
index 0000000000..47aaad5177
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ArchiveTimeBoundsTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pkg;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.time.Instant;
+import java.util.Date;
+
+import org.apache.commons.compress.archivers.zip.X000A_NTFS;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.FileSystem;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * Archive header times are binary fields; a junk value (e.g. a garbage FILETIME) must not
+ * reach fs:* as an impossible year that Metadata.getDate() cannot read back.
+ */
+public class ArchiveTimeBoundsTest {
+
+ private static final Date YEAR_30828 = Date.from(Instant.parse("+30828-09-14T02:48:05Z"));
+ private static final Date YEAR_0999 = Date.from(Instant.parse("0999-01-01T00:00:00Z"));
+ private static final Date VALID = Date.from(Instant.parse("2020-01-02T03:04:05Z"));
+
+ @Test
+ public void testInstantsOutOfBoundsDropped() {
+ Metadata m = new Metadata();
+ AbstractArchiveParser.setInstant(m, FileSystem.CREATED, YEAR_30828);
+ AbstractArchiveParser.setInstant(m, FileSystem.ACCESSED, YEAR_0999);
+ AbstractArchiveParser.setInstant(m, FileSystem.MODIFIED, VALID);
+ assertNull(m.get(FileSystem.CREATED));
+ assertNull(m.get(FileSystem.ACCESSED));
+ assertEquals("2020-01-02T03:04:05Z", m.get(FileSystem.MODIFIED));
+ }
+
+ @Test
+ public void testLocalTimeOutOfBoundsDropped() {
+ Metadata m = new Metadata();
+ AbstractArchiveParser.setLocalTime(m, FileSystem.CREATED, YEAR_30828);
+ assertNull(m.get(FileSystem.CREATED));
+ }
+
+ // X5455 is 32-bit Unix time and cannot hold a junk year; the NTFS FILETIME field can
+ @Test
+ public void testZipNtfsOutOfBoundsDropped() {
+ ZipArchiveEntry entry = new ZipArchiveEntry("a.txt");
+ X000A_NTFS ntfs = new X000A_NTFS();
+ ntfs.setModifyJavaTime(VALID);
+ ntfs.setAccessJavaTime(Date.from(Instant.parse("2500-01-01T00:00:00Z")));
+ ntfs.setCreateJavaTime(YEAR_0999);
+ entry.addExtraField(ntfs);
+ Metadata m = new Metadata();
+ ZipParser.setEntryTimes(entry, m);
+ assertEquals("2020-01-02T03:04:05Z", m.get(FileSystem.MODIFIED));
+ assertNull(m.get(FileSystem.ACCESSED));
+ assertNull(m.get(FileSystem.CREATED));
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/GzipParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/GzipParserTest.java
index 3dd16378ab..a18ec01848 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/GzipParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/GzipParserTest.java
@@ -17,21 +17,50 @@
package org.apache.tika.parser.pkg;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
import java.util.List;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipParameters;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.apache.tika.TikaTest;
+import org.apache.tika.metadata.FileSystem;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
/**
* Test case for parsing gzip files.
*/
public class GzipParserTest extends TikaTest {
+ @Test
+ public void testHeaderMtimeIsFileSystemModified(@TempDir Path tempDir) throws Exception {
+ assertEquals("2012-02-20T16:44:22Z", entryModified(tempDir, Instant.parse("2012-02-20T16:44:22Z")));
+ assertNull(entryModified(tempDir, Instant.EPOCH)); // MTIME 0: no timestamp
+ }
+
+ private String entryModified(Path tempDir, Instant mtime) throws Exception {
+ Path gz = tempDir.resolve("mtime-" + mtime.getEpochSecond() + ".gz");
+ GzipParameters params = new GzipParameters();
+ params.setFileName("a.txt");
+ params.setModificationInstant(mtime);
+ try (GzipCompressorOutputStream out = new GzipCompressorOutputStream(Files.newOutputStream(gz), params)) {
+ out.write("hello".getBytes(StandardCharsets.US_ASCII));
+ }
+ List<Metadata> list = getRecursiveMetadata(gz, new ParseContext(), false);
+ assertNull(list.get(1).get(TikaCoreProperties.MODIFIED));
+ return list.get(1).get(FileSystem.MODIFIED);
+ }
+
/**
* Tests that the ParseContext parser is correctly
* fired for all the embedded entries.
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipDosTimeDstTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipDosTimeDstTest.java
new file mode 100644
index 0000000000..b1fef4dc19
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipDosTimeDstTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pkg;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Isolated;
+import org.junit.jupiter.api.parallel.ResourceLock;
+import org.junit.jupiter.api.parallel.Resources;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.FileSystem;
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * A DOS time inside the host's DST spring-forward gap: commons-compress resolves it in the JVM
+ * zone and moves 02:30 to 03:30; the zip path reads the raw header field instead.
+ */
+@Isolated
+@ResourceLock(Resources.TIME_ZONE)
+public class ZipDosTimeDstTest extends TikaTest {
+
+ private static TimeZone saved;
+
+ @BeforeAll
+ public static void berlin() {
+ saved = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("Europe/Berlin"));
+ }
+
+ @AfterAll
+ public static void restore() {
+ TimeZone.setDefault(saved);
+ }
+
+ @Test
+ public void testDosTimeInDstGap() throws Exception {
+ List<Metadata> list = getRecursiveMetadata(TikaInputStream.get(zipAt(2021, 3, 28, 2, 30, 10)),
+ new ZipParser(), false);
+ assertEquals("2021-03-28T02:30:10", list.get(1).get(FileSystem.MODIFIED));
+ }
+
+ @Test
+ public void testDosTimeOutsideGap() throws Exception {
+ List<Metadata> list = getRecursiveMetadata(TikaInputStream.get(zipAt(2021, 7, 1, 9, 15, 0)),
+ new ZipParser(), false);
+ assertEquals("2021-07-01T09:15:00", list.get(1).get(FileSystem.MODIFIED));
+ }
+
+ // java.util.zip writes the DOS fields; patch both headers with the exact wall clock
+ private static byte[] zipAt(int year, int month, int day, int hour, int min, int sec) throws Exception {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try (ZipOutputStream zos = new ZipOutputStream(bos)) {
+ zos.putNextEntry(new ZipEntry("a.txt"));
+ zos.write("hello".getBytes(StandardCharsets.US_ASCII));
+ zos.closeEntry();
+ }
+ byte[] b = bos.toByteArray();
+ int time = (hour << 11) | (min << 5) | (sec / 2);
+ int date = ((year - 1980) << 9) | (month << 5) | day;
+ patch(b, 10, time, date);
+ for (int i = 0; i + 4 < b.length; i++) {
+ if (b[i] == 0x50 && b[i + 1] == 0x4b && b[i + 2] == 1 && b[i + 3] == 2) {
+ patch(b, i + 12, time, date);
+ }
+ }
+ return b;
+ }
+
+ private static void patch(byte[] b, int off, int time, int date) {
+ b[off] = (byte) time;
+ b[off + 1] = (byte) (time >> 8);
+ b[off + 2] = (byte) date;
+ b[off + 3] = (byte) (date >> 8);
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipEntryTimesTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipEntryTimesTest.java
new file mode 100644
index 0000000000..3578cf7458
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipEntryTimesTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pkg;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.parallel.Isolated;
+import org.junit.jupiter.api.parallel.ResourceLock;
+import org.junit.jupiter.api.parallel.Resources;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.metadata.FileSystem;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+
+/** Runs in a +14h zone. */
+@Isolated
+@ResourceLock(Resources.TIME_ZONE)
+public class ZipEntryTimesTest extends TikaTest {
+
+ private static TimeZone originalTimeZone;
+
+ @BeforeAll
+ static void init() {
+ originalTimeZone = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("Pacific/Kiritimati"));
+ }
+
+ @AfterAll
+ static void tearDown() {
+ TimeZone.setDefault(originalTimeZone);
+ }
+
+ @Test
+ public void testEntryTimesAreFileSystemTimes(@TempDir Path tempDir) throws Exception {
+ Path zip = tempDir.resolve("times.zip");
+ try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(Files.newOutputStream(zip))) {
+ ZipArchiveEntry dos = new ZipArchiveEntry("dos.txt");
+ dos.setTime(LocalDateTime.of(2009, 8, 11, 9, 9, 44)
+ .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
+ zos.putArchiveEntry(dos);
+ zos.write("dos".getBytes(StandardCharsets.US_ASCII));
+ zos.closeArchiveEntry();
+
+ ZipArchiveEntry ext = new ZipArchiveEntry("ext.txt");
+ X5455_ExtendedTimestamp ts = new X5455_ExtendedTimestamp();
+ ts.setModifyJavaTime(Date.from(Instant.parse("2012-02-20T16:44:22Z")));
+ ext.addExtraField(ts);
+ zos.putArchiveEntry(ext);
+ zos.write("ext".getBytes(StandardCharsets.US_ASCII));
+ zos.closeArchiveEntry();
+ }
+ List<Metadata> list = getRecursiveMetadata(zip, new ParseContext(), false);
+ Metadata dosMd = entry(list, "dos.txt");
+ Metadata extMd = entry(list, "ext.txt");
+ assertEquals("2009-08-11T09:09:44", dosMd.get(FileSystem.MODIFIED));
+ assertEquals("2012-02-20T16:44:22Z", extMd.get(FileSystem.MODIFIED));
+ for (Metadata m : List.of(dosMd, extMd)) {
+ assertNull(m.get(TikaCoreProperties.CREATED));
+ assertNull(m.get(TikaCoreProperties.MODIFIED));
+ }
+ }
+
+ private static Metadata entry(List<Metadata> list, String name) {
+ return list.stream().filter(m -> name.equals(m.get(TikaCoreProperties.RESOURCE_NAME_KEY)))
+ .findFirst().orElseThrow();
+ }
+}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java
index dc161cbfa8..9abf2af6e9 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xml/MetadataHandler.java
@@ -59,7 +59,9 @@ public class MetadataHandler extends DefaultHandler {
public void addMetadata(String value) {
if (value.length() > 0) {
String previous = metadata.get(name);
- if (previous != null && previous.length() > 0) {
+ // a joined date is unparseable: replace instead
+ if (previous != null && previous.length() > 0
+ && property.getPrimaryProperty().getValueType() != Property.ValueType.DATE) {
value = previous + ", " + value;
}
metadata.set(property, value);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/pom.xml
index b7a1fd656d..d1581c1d93 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/pom.xml
@@ -29,17 +29,6 @@
<artifactId>tika-parser-xmp-commons</artifactId>
<name>Apache Tika XMP commons</name>
-
- <dependencies>
- <!-- pdfbox-core for org.apache.pdfbox.util.DateConverter (XMP date normalization).
- Already on the standard-bundle classpath via the PDF module; net-new only to
- standalone image/IDML consumers. See the XMP-unification plan. -->
- <dependency>
- <groupId>org.apache.pdfbox</groupId>
- <artifactId>pdfbox</artifactId>
- <version>${pdfbox.version}</version>
- </dependency>
- </dependencies>
<build>
<plugins>
<plugin>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpDates.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpDates.java
index 92f5f6ace2..c8760db15e 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpDates.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpDates.java
@@ -16,45 +16,15 @@
*/
package org.apache.tika.parser.xmp;
-import java.util.Calendar;
-import java.util.Date;
+import org.apache.tika.utils.TikaDates;
-import org.apache.pdfbox.util.DateConverter;
-
-import org.apache.tika.utils.DateUtils;
-
-/** XMP date string -> canonical ISO-8601 UTC (seconds), or null if unrecognizable. */
+/** Delegates to {@link TikaDates#toMetadataString(String)}. */
public final class XmpDates {
private XmpDates() {
}
public static String normalize(String raw) {
- if (raw == null) {
- return null;
- }
- String s = raw.trim();
- if (s.isEmpty()) {
- return null;
- }
- // Known gap: a partial date (YYYY, YYYY-MM) inflates to a full timestamp -- preserving it
- // breaks Metadata.getDate() (can't re-parse year/month-only). Deferred to the
- // value-representation ticket. PDFBox handles PDF D: dates, producer formats, robust TZ.
- try {
- Calendar c = DateConverter.toCalendar(s);
- if (c != null) {
- return DateUtils.formatDate(c);
- }
- } catch (SecurityException e) {
- throw e;
- } catch (Exception e) {
- // fall through
- }
- // DateUtils fallback catches EXIF yyyy:MM:dd; not thread-safe, so new instance.
- Date d = new DateUtils().tryToParse(s);
- if (d != null) {
- return DateUtils.formatDate(d);
- }
- return null;
+ return TikaDates.toMetadataString(raw);
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java
index 272c01d35f..858220c204 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/main/java/org/apache/tika/parser/xmp/XmpExtractor.java
@@ -416,7 +416,10 @@ public class XmpExtractor {
boolean topLevel = p.path.indexOf('/') < 0;
Property fill = FILL_IF_ABSENT.get(uri + " " + ln);
if (topLevel && fill != null && metadata.get(fill) == null) {
- metadata.set(fill, valueFor(fill, value)); // canonical date, only if none yet
+ String norm = XmpDates.normalize(value); // fill-if-absent, never raw
+ if (norm != null) {
+ metadata.set(fill, norm);
+ }
}
// Language alternatives (leaves with an xml:lang) are handled in emitLangAlternatives().
Property[] props = TABLE.get(uri + " " + ln);
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpDatesTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpDatesTest.java
index e13fa2b4bf..243436db22 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpDatesTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpDatesTest.java
@@ -49,9 +49,18 @@ public class XmpDatesTest {
/** PDF D: form and date-only forms are recognized. */
@Test
public void testOtherRecognizedForms() {
- assertNotNull(XmpDates.normalize("D:20030101120000+05'30'"));
- assertNotNull(XmpDates.normalize("2015-06-12"));
- assertNotNull(XmpDates.normalize("2015:06:12")); // EXIF date-only, via DateUtils fallback
+ assertEquals("2003-01-01T06:30:00Z", XmpDates.normalize("D:20030101120000+05'30'"));
+ assertEquals("2015-06-12", XmpDates.normalize("2015-06-12")); // date-only stays a date
+ assertEquals("2015-06-12", XmpDates.normalize("2015:06:12"));
+ assertEquals("2007-10-06T16:27:07", XmpDates.normalize("2007:10:06 16:27:07"));
+ }
+
+ @Test
+ public void testZonelessStaysZoneless() {
+ assertEquals("2010-01-10T13:00:19", XmpDates.normalize("2010-01-10T13:00:19"));
+ assertEquals("2013-09-08T04:14:06", XmpDates.normalize("2013-09-08T04:14:06.006"));
+ assertEquals("2015-08-26T09:39:00", XmpDates.normalize("2015-08-26T09:39"));
+ assertEquals("2004-10-28T18:46:21", XmpDates.normalize("D:20041028184621"));
}
/** Non-dates degrade to null so the caller can pass the raw value through. */
@@ -62,13 +71,17 @@ public class XmpDatesTest {
assertNull(XmpDates.normalize(null));
}
- /**
- * A partial date (YYYY, YYYY-MM) inflates to a full timestamp; exact-equality (not assertNotNull)
- * pins that fabricated precision so the value-representation fix trips this test.
- */
@Test
- public void testPartialDatesInflateToFullTimestamp() {
- assertEquals("2019-01-01T00:00:00Z", XmpDates.normalize("2019"));
- assertEquals("2019-06-01T00:00:00Z", XmpDates.normalize("2019-06"));
+ public void testYearZeroReturnsNull() {
+ assertNull(XmpDates.normalize("0-01-01T00:00:00Z"));
+ assertNull(XmpDates.normalize("0-00-00T00:00:00Z"));
+ assertNull(XmpDates.normalize("0-00-00T00:00:00-04:00"));
+ assertNull(XmpDates.normalize("0000-01-01T00:00:00Z"));
+ }
+
+ @Test
+ public void testPartialDatesReturnNull() {
+ assertNull(XmpDates.normalize("2019"));
+ assertNull(XmpDates.normalize("2019-06"));
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpExtractorTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpExtractorTest.java
index edb5e3c457..771f5c79d5 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpExtractorTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xmp-commons/src/test/java/org/apache/tika/parser/xmp/XmpExtractorTest.java
@@ -93,6 +93,29 @@ public class XmpExtractorTest {
assertEquals("2020-01-02T03:04:05Z", md.get(XMP.CREATE_DATE)); // xmp key still set
}
+ @Test
+ public void testYearZeroNotPromotedToCreated() throws Exception {
+ Metadata md = new Metadata();
+ new XmpExtractor().extract(PACKET.replace("2020-01-02T03:04:05Z", "0-01-01T00:00:00Z")
+ .getBytes(UTF_8), md);
+ assertNull(md.get(TikaCoreProperties.CREATED));
+ assertEquals("0-01-01T00:00:00Z", md.get(XMP.CREATE_DATE)); // raw value kept on the xmp key
+ }
+
+ @Test
+ public void testZonelessDatesStayZoneless() throws Exception {
+ Metadata md = new Metadata();
+ new XmpExtractor().extract(PACKET.replace("2020-01-02T03:04:05Z", "2010-01-10T13:00:19")
+ .getBytes(UTF_8), md);
+ assertEquals("2010-01-10T13:00:19", md.get(XMP.CREATE_DATE));
+ assertEquals("2010-01-10T13:00:19", md.get(TikaCoreProperties.CREATED));
+
+ md = new Metadata();
+ new XmpExtractor().extract(PACKET.replace("2020-01-02T03:04:05Z", "2015:06:12")
+ .getBytes(UTF_8), md);
+ assertEquals("2015-06-12", md.get(TikaCoreProperties.CREATED));
+ }
+
@Test
public void testMultiValued() {
assertArrayEquals(new String[]{"Alice", "Bob"}, metadata.getValues(TikaCoreProperties.CREATOR));
diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
index b88db8e1f9..9b4ab0f411 100644
--- a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
+++ b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitter.java
@@ -25,16 +25,15 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Date;
+import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.TimeZone;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
@@ -50,6 +49,7 @@ import org.apache.tika.pipes.api.emitter.AbstractEmitter;
import org.apache.tika.pipes.api.emitter.EmitData;
import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.TikaDates;
/**
* Emitter to write parsed documents to a JDBC database.
@@ -82,9 +82,8 @@ import org.apache.tika.utils.StringUtils;
public class JDBCEmitter extends AbstractEmitter implements Closeable {
private static final Logger LOGGER = LoggerFactory.getLogger(JDBCEmitter.class);
+ private static final TimeZone UTC = TimeZone.getTimeZone("UTC");
- private static final String[] TIKA_DATE_PATTERNS =
- new String[]{"yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss"};
private static final ReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock();
private static final Set<String> TABLES_CREATED = new HashSet<>();
@@ -92,7 +91,6 @@ public class JDBCEmitter extends AbstractEmitter implements Closeable {
private final JDBCEmitterConfig.AttachmentStrategy attachmentStrategy;
private final JDBCEmitterConfig.MultivaluedFieldStrategy multivaluedFieldStrategy;
private final List<ColumnDefinition> columns;
- private final DateFormat[] dateFormats;
private final StringNormalizer stringNormalizer;
private Connection connection;
@@ -110,10 +108,6 @@ public class JDBCEmitter extends AbstractEmitter implements Closeable {
this.attachmentStrategy = config.getAttachmentStrategyEnum();
this.multivaluedFieldStrategy = config.getMultivaluedFieldStrategyEnum();
this.columns = parseColTypes(config);
- this.dateFormats = new DateFormat[TIKA_DATE_PATTERNS.length];
- for (int i = 0; i < TIKA_DATE_PATTERNS.length; i++) {
- dateFormats[i] = new SimpleDateFormat(TIKA_DATE_PATTERNS[i], Locale.US);
- }
this.stringNormalizer = config.connection().startsWith("jdbc:postgres")
? new PostgresNormalizer() : new StringNormalizer();
@@ -309,7 +303,7 @@ public class JDBCEmitter extends AbstractEmitter implements Closeable {
updateDouble(insertStatement, i, val);
break;
case Types.TIMESTAMP:
- updateTimestamp(insertStatement, i, val, dateFormats);
+ updateTimestamp(insertStatement, i, val);
break;
default:
throw new IllegalArgumentException("Can only process: " + getHandledTypes() +
@@ -366,23 +360,20 @@ public class JDBCEmitter extends AbstractEmitter implements Closeable {
insertStatement.setString(i, normalized);
}
- private void updateTimestamp(PreparedStatement insertStatement, int i, String val,
- DateFormat[] dateFormats) throws SQLException {
+ // zone-less values are UTC and date-only values midday UTC, as Metadata.getDate() reads them;
+ // the column gets the UTC wall clock whatever the JVM zone
+ private void updateTimestamp(PreparedStatement insertStatement, int i, String val) throws SQLException {
if (StringUtils.isBlank(val)) {
insertStatement.setNull(i, Types.TIMESTAMP);
return;
}
- for (DateFormat df : dateFormats) {
- try {
- Date d = df.parse(val);
- insertStatement.setTimestamp(i, new Timestamp(d.getTime()));
- return;
- } catch (ParseException e) {
- // ignore
- }
+ Optional<TikaDates.ParsedDate> d = TikaDates.parse(val).filter(TikaDates.ParsedDate::isFullPrecision);
+ if (d.isEmpty()) {
+ LOGGER.warn("Couldn't parse {}", val);
+ insertStatement.setNull(i, Types.TIMESTAMP);
+ return;
}
- LOGGER.warn("Couldn't parse {}", val);
- insertStatement.setNull(i, Types.TIMESTAMP);
+ insertStatement.setTimestamp(i, Timestamp.from(d.get().toInstant()), Calendar.getInstance(UTC, Locale.ROOT));
}
private void updateFloat(PreparedStatement insertStatement, int i, String val) throws SQLException {
diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
index a5195e7902..7dfcfaa452 100644
--- a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
+++ b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.pipes.emitter.jdbc;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
@@ -29,11 +30,13 @@ import java.sql.Statement;
import java.sql.Timestamp;
import java.time.ZoneId;
import java.util.ArrayList;
+import java.util.Calendar;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.TimeZone;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -144,6 +147,42 @@ public class JDBCEmitterTest {
}
}
+ @Test
+ public void testTimestampForms(@TempDir Path tmpDir) throws Exception {
+ Files.createDirectories(tmpDir.resolve("db"));
+ String connectionString = "jdbc:h2:file:" + tmpDir.resolve("db/h2").toAbsolutePath();
+ LinkedHashMap<String, String> keys = new LinkedHashMap<>();
+ keys.put("k6", "timestamp");
+ ObjectNode configNode = createConfigNode(connectionString,
+ "insert into test (path, k6) values (?,?)",
+ "create table test (path varchar(512) primary key, k6 timestamp)",
+ null, "first_only", keys);
+ JDBCEmitter emitter = JDBCEmitter.build(new ExtensionConfig("test-jdbc", "jdbc-emitter",
+ MAPPER.writeValueAsString(configNode)));
+ // stored form -> expected UTC instant (zone-less is UTC, date-only is midday UTC)
+ String[][] cases = {
+ {"2022-11-04T17:10:15Z", "2022-11-04T17:10:15Z"},
+ {"2022-11-04T17:10:15", "2022-11-04T17:10:15Z"},
+ {"2022-11-04", "2022-11-04T12:00:00Z"},
+ {"2022-11-04T19:10:15+02:00", "2022-11-04T17:10:15Z"},
+ };
+ for (int i = 0; i < cases.length; i++) {
+ emitter.emit("id" + i, Collections.singletonList(m(new String[]{"k6", cases[i][0]})),
+ new ParseContext());
+ }
+ Calendar utc = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+ try (Connection connection = DriverManager.getConnection(connectionString);
+ Statement st = connection.createStatement();
+ ResultSet rs = st.executeQuery("select path, k6 from test order by path")) {
+ for (String[] c : cases) {
+ assertTrue(rs.next());
+ Timestamp ts = rs.getTimestamp(2, utc);
+ assertNotNull(ts, c[0]);
+ assertEquals(c[1], ts.toInstant().toString(), c[0]);
+ }
+ }
+ }
+
@Test
public void testTableExists(@TempDir Path tmpDir) throws Exception {
String createTable = "create table test (path varchar(512) primary key," +
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/TestMetadataFilter.java b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/TestMetadataFilter.java
index a74b253d11..d027329206 100644
--- a/tika-serialization/src/test/java/org/apache/tika/metadata/filter/TestMetadataFilter.java
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/filter/TestMetadataFilter.java
@@ -214,6 +214,59 @@ public class TestMetadataFilter extends TikaTest {
assertEquals("2021-07-23T08:02:24Z", m.get(TikaCoreProperties.CREATED));
}
+ @Test
+ public void testDateNormalizingFilterKeepsOffset() throws Exception {
+ DateNormalizingMetadataFilter filter = new DateNormalizingMetadataFilter();
+ filter.setDefaultTimeZone("America/Los_Angeles");
+ String[][] cases = {
+ {"2010-05-09T21:34:38+0200", "2010-05-09T19:34:38Z"},
+ {"2010-05-09T21:34:38+02:00", "2010-05-09T19:34:38Z"},
+ {"2010-05-09T21:34:38-05:00", "2010-05-10T02:34:38Z"},
+ {"2010-05-09T21:34:38.123+02:00", "2010-05-09T19:34:38Z"},
+ };
+ for (String[] c : cases) {
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.CREATED, c[0]);
+ filter.filter(m);
+ assertEquals(c[1], m.get(TikaCoreProperties.CREATED), c[0]);
+ }
+ }
+
+ @Test
+ public void testDateNormalizingFilterStoredForms() throws Exception {
+ DateNormalizingMetadataFilter filter = new DateNormalizingMetadataFilter();
+ filter.setDefaultTimeZone("America/New_York");
+ String[][] cases = {
+ // date-only reads as midday, like Metadata.getDate(), in the default zone
+ {"2015-06-12", "2015-06-12T16:00:00Z"},
+ {"2020-01-01T10:00:00Z00:00", "2020-01-01T10:00:00Z"},
+ // partials: missing fields take their minimum, midday in the default zone
+ {"2018", "2018-01-01T17:00:00Z"},
+ {"2018-06", "2018-06-01T16:00:00Z"},
+ // unparseable values are left as they are, without an exception
+ {"garbage", "garbage"},
+ {"+999999999-12-31T23:59:59-10:00", "+999999999-12-31T23:59:59-10:00"},
+ };
+ for (String[] c : cases) {
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.CREATED, c[0]);
+ filter.filter(m);
+ assertEquals(c[1], m.get(TikaCoreProperties.CREATED), c[0]);
+ }
+ }
+
+ @Test
+ public void testDateNormalizingFilterMultiValued() throws Exception {
+ DateNormalizingMetadataFilter filter = new DateNormalizingMetadataFilter();
+ Metadata m = new Metadata();
+ m.add(TikaCoreProperties.SIGNATURE_DATE, "2010-05-09T21:34:38+02:00");
+ m.add(TikaCoreProperties.SIGNATURE_DATE, "2011-01-01T00:00:00Z");
+ m.add(TikaCoreProperties.SIGNATURE_DATE, "2012-06-01T08:00:00");
+ filter.filter(m);
+ assertArrayEquals(new String[]{"2010-05-09T19:34:38Z", "2011-01-01T00:00:00Z", "2012-06-01T08:00:00Z"},
+ m.getValues(TikaCoreProperties.SIGNATURE_DATE));
+ }
+
@Test
public void testCaptureGroupBasic() throws Exception {
TikaLoader loader = TikaLoader.load(getConfigPath(getClass(), "TIKA-4133-capture-group.json"));