You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2025/02/26 17:42:02 UTC
(tika) branch TIKA-4389 created (now ddbca884d)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch TIKA-4389
in repository https://gitbox.apache.org/repos/asf/tika.git
at ddbca884d TIKA-4389 cleanups for TIKA-4381
This branch includes the following new commits:
new ddbca884d TIKA-4389 cleanups for TIKA-4381
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 01/01: TIKA-4389 cleanups for TIKA-4381
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4389
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ddbca884d13886c88beca39be003ededcdadb7ef
Author: tallison <ta...@apache.org>
AuthorDate: Wed Feb 26 12:41:48 2025 -0500
TIKA-4389 cleanups for TIKA-4381
---
.../main/java/org/apache/tika/metadata/MAPI.java | 2 +-
.../microsoft/msg/ExtendedMetadataExtractor.java | 126 +++++++++++----------
.../parser/microsoft/msg/TikaNameIdChunks.java | 6 +-
.../tika/parser/microsoft/OutlookParserTest.java | 47 +++++---
4 files changed, 104 insertions(+), 77 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java b/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java
index 45fbad1bc..5f4ef12ae 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/MAPI.java
@@ -26,7 +26,7 @@ public interface MAPI {
String PREFIX_MAPI_META = "mapi" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
String PREFIX_MAPI_ATTACH_META = "mapi:attach" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
- String PREFIX_MAPI_RAW_META = PREFIX_MAPI_META + "raw" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
+ String PREFIX_MAPI_PROPERTY = PREFIX_MAPI_META + "property" + TikaCoreProperties.NAMESPACE_PREFIX_DELIMITER;
/**
* MAPI message class. What type of .msg/MAPI file is it?
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java
index 6d4880c51..5bf00f857 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/ExtendedMetadataExtractor.java
@@ -42,7 +42,6 @@ import org.slf4j.LoggerFactory;
import org.apache.tika.metadata.MAPI;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.microsoft.OutlookExtractor;
import org.apache.tika.utils.StringUtils;
/**
@@ -59,54 +58,14 @@ public class ExtendedMetadataExtractor {
loadProperties();
}
-
- private static List<Types.MAPIType> parseDataTypes(String[] arr) {
- if (arr.length == 1) {
- Types.MAPIType type = parseDataType(arr[0]);
- if (type != null) {
- return List.of(type);
- }
- return Collections.EMPTY_LIST;
- }
- List<Types.MAPIType> types = new ArrayList<>();
- for (String s : arr) {
- Types.MAPIType type = parseDataType(s);
- if (type != null) {
- types.add(type);
- }
- }
- return types;
- }
-
- private static Types.MAPIType parseDataType(String s) {
- if (StringUtils.isBlank(s)) {
- return null;
- }
- String[] parts = s.split(", ");
- if (parts.length != 2) {
- throw new IllegalArgumentException("expected two parts: " + s);
- }
- String num = parts[1];
- if (num.startsWith("0x")) {
- num = num.substring(2);
+ public static void extract(MAPIMessage msg, Metadata metadata) {
+ if (msg.getNameIdChunks() == null) {
+ return;
}
- int id = Integer.parseInt(num, 16);
- Types.MAPIType type = Types.getById(id);
- if (type == null) {
- //TODO:
- /*
- PtypRestriction, 0x00FD
- PtypRuleAction, 0x00FE
- PtypServerId, 0x00FB
- */
- return Types.createCustom(id);
+ if (msg.getMainChunks() == null || msg.getMainChunks().getRawProperties() == null) {
+ return;
}
- return type;
- }
-
-
- public static void extract(MAPIMessage msg, Metadata metadata) {
- //prep our custom nameIdchunk handler
+ //prep our custom nameIdChunk handler
TikaNameIdChunks tikaNameIdChunks = new TikaNameIdChunks();
//short-circuit for files that have an empty nameIdChunk
long len = 0;
@@ -124,7 +83,11 @@ public class ExtendedMetadataExtractor {
if (len == 0) {
return;
}
- tikaNameIdChunks.chunksComplete();
+ try {
+ tikaNameIdChunks.chunksComplete();
+ } catch (IllegalStateException e) {
+ LOGGER.warn("bad namechunks stream", e);
+ }
for (Map.Entry<MAPIProperty, PropertyValue> e : msg
.getMainChunks()
.getRawProperties()
@@ -132,6 +95,9 @@ public class ExtendedMetadataExtractor {
//the mapiproperties from POI are the literal storage id for that particular file.
//Those storage ids must be mapped via the name chunk ids into a known id
PropertyValue v = e.getValue();
+ if (v == null) {
+ continue;
+ }
List<MAPITag> mapiTags = tikaNameIdChunks.getTags(e.getKey().id);
MAPITagPair pair = null;
for (MAPITag mapiTag : mapiTags) {
@@ -146,7 +112,6 @@ public class ExtendedMetadataExtractor {
}
updateMetadata(pair, v, metadata);
}
-
}
@@ -180,7 +145,7 @@ public class ExtendedMetadataExtractor {
if (!includeType(propertyValue)) {
return;
}
- String key = MAPI.PREFIX_MAPI_RAW_META + pair.tikaMapiProperty.name;
+ String key = MAPI.PREFIX_MAPI_PROPERTY + pair.tikaMapiProperty.name;
Types.MAPIType type = propertyValue.getActualType();
if (type == Types.TIME || type == Types.MV_TIME || type == Types.APP_TIME || type == Types.MV_APP_TIME) {
Calendar calendar = (Calendar) propertyValue.getValue();
@@ -190,8 +155,12 @@ public class ExtendedMetadataExtractor {
.toString();
metadata.add(key, calendarString);
} else if (type == Types.BOOLEAN) {
- metadata.add(key, Boolean.toString((boolean) propertyValue.getValue()));
- } else {
+ Boolean val = (Boolean)propertyValue.getValue();
+ if (val == null) {
+ return;
+ }
+ metadata.add(key, Boolean.toString(val));
+ } else if (! StringUtils.isBlank(propertyValue.toString())) {
metadata.add(key, propertyValue.toString());
}
@@ -205,11 +174,6 @@ public class ExtendedMetadataExtractor {
return true;
}
- private static boolean isString(PropertyValue propertyValue) {
- Types.MAPIType mapiType = propertyValue.getActualType();
- return mapiType == Types.ASCII_STRING || mapiType == Types.MV_ASCII_STRING || mapiType == Types.MV_UNICODE_STRING || mapiType == Types.UNICODE_STRING;
- }
-
private static class TikaMapiProperty {
String name;
ClassID classID; // can be null
@@ -237,7 +201,7 @@ public class ExtendedMetadataExtractor {
.toUUIDString(), setType.getClassID());
}
try (BufferedReader r = new BufferedReader(
- new InputStreamReader(OutlookExtractor.class.getResourceAsStream("/org/apache/tika/parser/microsoft/msg/props_table.txt"), UTF_8))) {
+ new InputStreamReader(ExtendedMetadataExtractor.class.getResourceAsStream("/org/apache/tika/parser/microsoft/msg/props_table.txt"), UTF_8))) {
String line = r.readLine();
while (line != null) {
if (line.isBlank() || line.startsWith("#")) {
@@ -309,4 +273,50 @@ public class ExtendedMetadataExtractor {
this.tikaMapiProperty = tikaMapiProperty;
}
}
+
+
+ private static List<Types.MAPIType> parseDataTypes(String[] arr) {
+ if (arr.length == 1) {
+ Types.MAPIType type = parseDataType(arr[0]);
+ if (type != null) {
+ return List.of(type);
+ }
+ return Collections.EMPTY_LIST;
+ }
+ List<Types.MAPIType> types = new ArrayList<>();
+ for (String s : arr) {
+ Types.MAPIType type = parseDataType(s);
+ if (type != null) {
+ types.add(type);
+ }
+ }
+ return types;
+ }
+
+ private static Types.MAPIType parseDataType(String s) {
+ if (StringUtils.isBlank(s)) {
+ return null;
+ }
+ String[] parts = s.split(", ");
+ if (parts.length != 2) {
+ throw new IllegalArgumentException("expected two parts: " + s);
+ }
+ String num = parts[1];
+ if (num.startsWith("0x")) {
+ num = num.substring(2);
+ }
+ int id = Integer.parseInt(num, 16);
+ Types.MAPIType type = Types.getById(id);
+ if (type == null) {
+ //TODO:
+ /*
+ PtypRestriction, 0x00FD
+ PtypRuleAction, 0x00FE
+ PtypServerId, 0x00FB
+ */
+ return Types.createCustom(id);
+ }
+ return type;
+ }
+
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/TikaNameIdChunks.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/TikaNameIdChunks.java
index 54e963ee3..ba54f6e4e 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/TikaNameIdChunks.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/msg/TikaNameIdChunks.java
@@ -18,6 +18,7 @@
package org.apache.tika.parser.microsoft.msg;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -132,10 +133,11 @@ public final class TikaNameIdChunks implements ChunkGroup {
loadTags();
}
+ //does not return null
public List<MAPITag> getTags(int storageId) {
List<MAPITag> tags = mapiTagMap.get(storageId);
if (tags == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
}
return tags;
}
@@ -235,7 +237,7 @@ public final class TikaNameIdChunks implements ChunkGroup {
return 0;
}
for (Chunk chunk : chunks) {
- if (chunk.getType() != Types.BINARY || chunk.getChunkId() != streamID) {
+ if (chunk == null || chunk.getType() != Types.BINARY || chunk.getChunkId() != streamID) {
continue;
}
byte[] matchChunkBytes = ((ByteChunk) chunk).getValue();
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
index 10d2e02b4..6c532a84e 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
@@ -312,30 +312,45 @@ public class OutlookParserTest extends TikaTest {
public void testAppointmentExtendedMetadata() throws Exception {
List<Metadata> metadataList = getRecursiveMetadata("testMSG_Appointment.msg");
Metadata m = metadataList.get(0);
- debug(m);
- assertTrue(m.get("mapi:raw:PidLidAppointmentEndWhole").contains("2017-02-28T19"));
- assertTrue(m.get("mapi:raw:PidLidAppointmentStartWhole").contains("2017-02-28T18"));
- assertTrue(m.get("mapi:raw:PidLidClipStart").contains("2017-02-28T18"));
- assertTrue(m.get("mapi:raw:PidLidClipEnd").contains("2017-02-28T19"));
- assertTrue(m.get("mapi:raw:PidLidCommonStart").contains("2017-02-28T18"));
- assertTrue(m.get("mapi:raw:PidLidCommonEnd").contains("2017-02-28T19"));
- assertTrue(m.get("mapi:raw:PidLidReminderSignalTime").contains("4501-01-01T00"));
- assertTrue(m.get("mapi:raw:PidLidReminderTime").contains("2017-02-28T18"));
- assertTrue(m.get("mapi:raw:PidLidValidFlagStringProof").contains("2017-02-28T18:42"));
- assertEquals("0", m.get("mapi:raw:PidLidAppointmentSequence"));
- assertEquals("false", m.get("mapi:raw:PidLidRecurring"));
+ assertTrue(m.get("mapi:property:PidLidAppointmentEndWhole").contains("2017-02-28T19"));
+ assertTrue(m.get("mapi:property:PidLidAppointmentStartWhole").contains("2017-02-28T18"));
+ assertTrue(m.get("mapi:property:PidLidClipStart").contains("2017-02-28T18"));
+ assertTrue(m.get("mapi:property:PidLidClipEnd").contains("2017-02-28T19"));
+ assertTrue(m.get("mapi:property:PidLidCommonStart").contains("2017-02-28T18"));
+ assertTrue(m.get("mapi:property:PidLidCommonEnd").contains("2017-02-28T19"));
+ assertTrue(m.get("mapi:property:PidLidReminderSignalTime").contains("4501-01-01T00"));
+ assertTrue(m.get("mapi:property:PidLidReminderTime").contains("2017-02-28T18"));
+ assertTrue(m.get("mapi:property:PidLidValidFlagStringProof").contains("2017-02-28T18:42"));
+ assertEquals("0", m.get("mapi:property:PidLidAppointmentSequence"));
+ assertEquals("false", m.get("mapi:property:PidLidRecurring"));
}
@Test
public void testTaskExtendedMetadata() throws Exception {
List<Metadata> metadataList = getRecursiveMetadata("testMSG_Task.msg");
Metadata m = metadataList.get(0);
- assertTrue(m.get("mapi:raw:PidLidToDoOrdinalDate").contains("2017-02-28T18:44"));
- assertTrue(m.get("mapi:raw:PidLidValidFlagStringProof").contains("2017-02-28T18:44"));
- assertEquals("0", m.get("mapi:raw:PidLidTaskActualEffort"));
- assertEquals("false", m.get("mapi:raw:PidLidTeamTask"));
+ assertTrue(m.get("mapi:property:PidLidToDoOrdinalDate").contains("2017-02-28T18:44"));
+ assertTrue(m.get("mapi:property:PidLidValidFlagStringProof").contains("2017-02-28T18:44"));
+ assertEquals("0", m.get("mapi:property:PidLidTaskActualEffort"));
+ assertEquals("false", m.get("mapi:property:PidLidTeamTask"));
}
+ @Test
+ public void testContactExtendedMetadata() throws Exception {
+ List<Metadata> metadataList = getRecursiveMetadata("testMSG_Contact.msg");
+ Metadata m = metadataList.get(0);
+ assertEquals("2017-02-28T18:41:37Z", m.get("mapi:property:PidLidValidFlagStringProof"));
+ }
+
+
+ @Test
+ public void testPostExtendedMetadata() throws Exception {
+ List<Metadata> metadataList = getRecursiveMetadata("testMSG_Post.msg");
+ Metadata m = metadataList.get(0);
+ assertEquals("2017-02-28T18:47:11Z", m.get("mapi:property:PidLidValidFlagStringProof"));
+ }
+
+
@Test
public void testHandlingAllAlternativesBodies() throws Exception {
//test that default only has one body