You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2024/08/05 16:41:35 UTC
(tika) 01/01: TIKA-4294 -- improve serialization of ParseContext
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4294
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 01b7b0da9c6066430c365955d33a044f31432f6e
Author: tallison <ta...@apache.org>
AuthorDate: Mon Aug 5 12:41:20 2024 -0400
TIKA-4294 -- improve serialization of ParseContext
---
.../serialization/ParseContextDeserializer.java | 79 ++++++++++++++++++++++
.../tika/serialization/ParseContextSerializer.java | 5 +-
.../tika/serialization/TikaJsonSerializer.java | 2 +-
.../serialization/pipes/JsonFetchEmitTuple.java | 34 ++--------
4 files changed, 88 insertions(+), 32 deletions(-)
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java
new file mode 100644
index 000000000..99cbf7645
--- /dev/null
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.serialization;
+
+import static org.apache.tika.serialization.ParseContextSerializer.PARSE_CONTEXT;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.commons.io.IOExceptionWithCause;
+
+import org.apache.tika.parser.ParseContext;
+
+public class ParseContextDeserializer extends JsonDeserializer<ParseContext> {
+
+ @Override
+ public ParseContext deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException {
+ JsonNode root = jsonParser.getCodec().readTree(jsonParser);
+ return readParseContext(root);
+ }
+
+ public static ParseContext readParseContext(JsonNode jsonNode) throws IOException {
+ //some use cases include the wrapper node, e.g. { "parseContext": {}}
+ //some include the contents only.
+ //Try to find "parseContext" to start. If that doesn't exist, assume the jsonNode is the contents.
+ JsonNode contextNode = jsonNode.get(PARSE_CONTEXT);
+
+ if (contextNode == null) {
+ contextNode = jsonNode;
+ }
+ ParseContext parseContext = new ParseContext();
+ Iterator<Map.Entry<String, JsonNode>> it = contextNode.fields();
+ while (it.hasNext()) {
+ Map.Entry<String, JsonNode> e = it.next();
+ String superClassName = e.getKey();
+ JsonNode obj = e.getValue();
+ String className = readVal(TikaJsonSerializer.INSTANTIATED_CLASS_KEY, obj, null, true);
+ try {
+ Class clazz = Class.forName(className);
+ Class superClazz = className.equals(superClassName) ? clazz : Class.forName(superClassName);
+ parseContext.set(clazz, TikaJsonDeserializer.deserialize(clazz, superClazz, obj));
+ } catch (ReflectiveOperationException ex) {
+ throw new IOExceptionWithCause(ex);
+ }
+ }
+ return parseContext;
+ }
+
+ private static String readVal(String key, JsonNode jsonObj, String defaultRet, boolean isRequired) throws IOException {
+ JsonNode valNode = jsonObj.get(key);
+ if (valNode == null) {
+ if (isRequired) {
+ throw new IOException("required value string, but see: " + key);
+ }
+ return defaultRet;
+ }
+ return valNode.asText();
+ }
+}
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
index ece9eb27b..95d6b3698 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
@@ -25,16 +25,17 @@ import com.fasterxml.jackson.databind.SerializerProvider;
import org.apache.tika.parser.ParseContext;
public class ParseContextSerializer extends JsonSerializer<ParseContext> {
+ public static final String PARSE_CONTEXT = "parseContext";
@Override
public void serialize(ParseContext parseContext, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
- jsonGenerator.writeFieldName("parseContext");
+ jsonGenerator.writeFieldName(PARSE_CONTEXT);
jsonGenerator.writeStartObject();
for (String className : parseContext.keySet()) {
try {
Class clazz = Class.forName(className);
- TikaJsonSerializer.serialize(className, parseContext.get(clazz), clazz, jsonGenerator);
+ TikaJsonSerializer.serialize(className, parseContext.get(clazz), null, jsonGenerator);
} catch (TikaSerializationException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
index 71e9c2342..b37f2c6e3 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
@@ -116,7 +116,7 @@ public class TikaJsonSerializer {
jsonGenerator.writeStringField(INSTANTIATED_CLASS_KEY, obj
.getClass()
.getName());
- if (!obj
+ if (superClass != null && !obj
.getClass()
.equals(superClass)) {
jsonGenerator.writeStringField(SUPER_CLASS_KEY, superClass.getName());
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/pipes/JsonFetchEmitTuple.java b/tika-serialization/src/main/java/org/apache/tika/serialization/pipes/JsonFetchEmitTuple.java
index 72082996b..48a16dd64 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/pipes/JsonFetchEmitTuple.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/pipes/JsonFetchEmitTuple.java
@@ -16,6 +16,8 @@
*/
package org.apache.tika.serialization.pipes;
+import static org.apache.tika.serialization.ParseContextSerializer.PARSE_CONTEXT;
+
import java.io.IOException;
import java.io.Reader;
import java.io.StringWriter;
@@ -28,7 +30,6 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOExceptionWithCause;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
@@ -36,9 +37,8 @@ import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.serialization.JsonMetadata;
+import org.apache.tika.serialization.ParseContextDeserializer;
import org.apache.tika.serialization.ParseContextSerializer;
-import org.apache.tika.serialization.TikaJsonDeserializer;
-import org.apache.tika.serialization.TikaJsonSerializer;
import org.apache.tika.utils.StringUtils;
public class JsonFetchEmitTuple {
@@ -52,7 +52,6 @@ public class JsonFetchEmitTuple {
public static final String EMITKEY = "emitKey";
public static final String METADATAKEY = "metadata";
public static final String ON_PARSE_EXCEPTION = "onParseException";
- public static final String PARSE_CONTEXT = "parseContext";
public static FetchEmitTuple fromJson(Reader reader) throws IOException {
//try (JsonParser jParser = new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
@@ -72,7 +71,8 @@ public class JsonFetchEmitTuple {
long fetchRangeStart = readLong(FETCH_RANGE_START, root, -1l, false);
long fetchRangeEnd = readLong(FETCH_RANGE_END, root, -1l, false);
Metadata metadata = readMetadata(root);
- ParseContext parseContext = readParseContext(root);
+ JsonNode parseContextNode = root.get(PARSE_CONTEXT);
+ ParseContext parseContext = parseContextNode == null ? new ParseContext() : ParseContextDeserializer.readParseContext(parseContextNode);
FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = readOnParseException(root);
return new FetchEmitTuple(id, new FetchKey(fetcherName, fetchKey, fetchRangeStart, fetchRangeEnd), new EmitKey(emitterName, emitKey), metadata, parseContext,
@@ -94,30 +94,6 @@ public class JsonFetchEmitTuple {
}
}
- private static ParseContext readParseContext(JsonNode root) throws IOException {
- JsonNode contextNode = root.get(PARSE_CONTEXT);
- if (contextNode == null) {
- return new ParseContext();
- }
- ParseContext parseContext = new ParseContext();
- Iterator<Map.Entry<String, JsonNode>> it = contextNode.fields();
- while (it.hasNext()) {
- Map.Entry<String, JsonNode> e = it.next();
- String clazzName = e.getKey();
- JsonNode obj = e.getValue();
- String className = readVal(TikaJsonSerializer.INSTANTIATED_CLASS_KEY, obj, null, true);
- String superClassName = readVal(TikaJsonSerializer.SUPER_CLASS_KEY, obj, className, false);
- try {
- Class clazz = Class.forName(className);
- Class superClazz = clazz.equals(superClassName) ? clazz : Class.forName(superClassName);
- parseContext.set(clazz, TikaJsonDeserializer.deserialize(clazz, superClazz, obj));
- } catch (ReflectiveOperationException ex) {
- throw new IOExceptionWithCause(ex);
- }
- }
- return parseContext;
- }
-
private static Metadata readMetadata(JsonNode root) {
JsonNode metadataNode = root.get(METADATAKEY);
if (metadataNode == null) {