You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2023/10/14 01:21:30 UTC
[tika] branch branch_2x updated: TIKA-4154 - parameterize max string length for jackson (#1402)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_2x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_2x by this push:
new 4f922cc1a TIKA-4154 - parameterize max string length for jackson (#1402)
4f922cc1a is described below
commit 4f922cc1a6d9159b9a06cc2b78e1b3bd5d393882
Author: Tim Allison <ta...@apache.org>
AuthorDate: Fri Oct 13 21:09:58 2023 -0400
TIKA-4154 - parameterize max string length for jackson (#1402)
* TIKA-4154 -- parameterize max string length for serialization with jackson
(cherry picked from commit e726aea123d37cbe1ed15b80c1d455da3e442ada)
---
.../java/org/apache/tika/config/TikaConfig.java | 32 +++++++++++++++++
.../tika/metadata/serialization/JsonEmitData.java | 7 +++-
.../metadata/serialization/JsonFetchEmitTuple.java | 5 ++-
.../serialization/JsonFetchEmitTupleList.java | 8 +++--
.../tika/metadata/serialization/JsonMetadata.java | 11 ++++--
.../serialization/JsonMetadataDeserializer.java | 42 ----------------------
.../metadata/serialization/JsonMetadataList.java | 15 +++++---
.../serialization/JsonMetadataSerializer.java | 40 ---------------------
.../serialization/JsonStreamingSerializer.java | 7 +++-
.../metadata/serialization/JsonMetadataTest.java | 13 +++++--
.../src/test/resources/config/tika-config-json.xml | 20 +++++++++++
.../tika/server/standard/TikaResourceTest.java | 29 +++++++++++++++
.../src/test/resources/config/tika-config-json.xml | 20 +++++++++++
13 files changed, 153 insertions(+), 96 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index cb23cec21..895ff048f 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -87,6 +87,9 @@ import org.apache.tika.utils.XMLReaderUtils;
*/
public class TikaConfig {
+ public static int DEFAULT_MAX_JSON_STRING_FIELD_LENGTH = 20_000_000;//jackson's default
+ public static String MAX_JSON_STRING_FIELD_LENGTH_ELEMENT_NAME = "maxJsonStringFieldLength";
+
//use this to look for unneeded instantiations of TikaConfig
protected static final AtomicInteger TIMES_INSTANTIATED = new AtomicInteger();
@@ -103,6 +106,8 @@ public class TikaConfig {
private final MetadataFilter metadataFilter;
private final AutoDetectParserConfig autoDetectParserConfig;
+ private static int MAX_JSON_STRING_FIELD_LENGTH = DEFAULT_MAX_JSON_STRING_FIELD_LENGTH;
+
public TikaConfig(String file) throws TikaException, IOException, SAXException {
this(Paths.get(file));
}
@@ -174,6 +179,7 @@ public class TikaConfig {
this.metadataFilter = MetadataFilter.load(element, true);
this.autoDetectParserConfig = AutoDetectParserConfig.load(element);
this.serviceLoader = loader;
+ setMaxJsonStringFieldLength(element);
TIMES_INSTANTIATED.incrementAndGet();
}
@@ -273,6 +279,7 @@ public class TikaConfig {
executorLoader.loadOverall(element, mimeTypes, serviceLoader);
this.metadataFilter = MetadataFilter.load(element, true);
this.autoDetectParserConfig = AutoDetectParserConfig.load(element);
+ setMaxJsonStringFieldLength(element);
} catch (SAXException e) {
throw new TikaException("Specified Tika configuration has syntax errors: " + config,
e);
@@ -281,6 +288,31 @@ public class TikaConfig {
TIMES_INSTANTIATED.incrementAndGet();
}
+ /**
+ *
+ * @return maximum field length when serializing String fields in Tika's metadata or metadata
+ * list into JSON
+ */
+ public static int getMaxJsonStringFieldLength() {
+ return MAX_JSON_STRING_FIELD_LENGTH;
+ }
+
+ private void setMaxJsonStringFieldLength(Element properties) throws TikaConfigException {
+ NodeList nodeList = properties.getChildNodes();
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ Node n = nodeList.item(i);
+ if (n.getNodeName().equals(MAX_JSON_STRING_FIELD_LENGTH_ELEMENT_NAME)) {
+ try {
+ MAX_JSON_STRING_FIELD_LENGTH = Integer.parseInt(n.getTextContent());
+ } catch (NumberFormatException e) {
+ throw new TikaConfigException(MAX_JSON_STRING_FIELD_LENGTH_ELEMENT_NAME + " " +
+ "is not an integer", e);
+ }
+ return;
+ }
+ }
+ }
+
private static MimeTypes getDefaultMimeTypes(ClassLoader loader) {
return MimeTypes.getDefaultMimeTypes(loader);
}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
index 095105ba6..cc9cd9320 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
@@ -21,7 +21,9 @@ import java.io.Writer;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.emitter.EmitData;
import org.apache.tika.pipes.emitter.EmitKey;
@@ -29,7 +31,10 @@ import org.apache.tika.pipes.emitter.EmitKey;
public class JsonEmitData {
public static void toJson(EmitData emitData, Writer writer) throws IOException {
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ try (JsonGenerator jsonGenerator = new JsonFactory()
+ .setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength())
+ .build()).createGenerator(writer)) {
jsonGenerator.writeStartObject();
EmitKey key = emitData.getEmitKey();
jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITTER, key.getEmitterName());
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
index 714610786..3fbd67c0c 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
@@ -26,7 +26,9 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.HandlerConfig;
@@ -54,7 +56,8 @@ public class JsonFetchEmitTuple {
public static FetchEmitTuple fromJson(Reader reader) throws IOException {
- try (JsonParser jParser = new JsonFactory().createParser(reader)) {
+ try (JsonParser jParser = new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build()).createParser(reader)) {
JsonToken token = jParser.nextToken();
if (token != JsonToken.START_OBJECT) {
throw new IOException("require start object, but see: " + token.name());
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java
index e74bd537f..e7181aeb3 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java
@@ -27,14 +27,17 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.pipes.FetchEmitTuple;
public class JsonFetchEmitTupleList {
public static List<FetchEmitTuple> fromJson(Reader reader) throws IOException {
List<FetchEmitTuple> list;
- try (JsonParser jParser = new JsonFactory().createParser(reader)) {
+ try (JsonParser jParser = new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build()).createParser(reader)) {
JsonToken token = jParser.nextToken();
if (token != JsonToken.START_ARRAY) {
throw new IOException("require start array, but see: " + token.name());
@@ -56,7 +59,8 @@ public class JsonFetchEmitTupleList {
public static void toJson(List<FetchEmitTuple> list, Writer writer) throws IOException {
- try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ try (JsonGenerator jsonGenerator = new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build()).createGenerator(writer)) {
jsonGenerator.writeStartArray();
for (FetchEmitTuple t : list) {
JsonFetchEmitTuple.writeTuple(t, jsonGenerator);
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java
index af55e3e92..2e7a931b2 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java
@@ -26,9 +26,11 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.StreamReadConstraints;
import org.apache.commons.io.input.CloseShieldReader;
import org.apache.commons.io.output.CloseShieldWriter;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
@@ -48,7 +50,10 @@ public class JsonMetadata {
writer.write("null");
return;
}
- try (JsonGenerator jsonGenerator = new JsonFactory()
+ long max = TikaConfig.getMaxJsonStringFieldLength();
+ try (JsonGenerator jsonGenerator = new JsonFactory().setStreamReadConstraints(
+ StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build())
.createGenerator(new CloseShieldWriter(writer))) {
if (PRETTY_PRINT) {
jsonGenerator.useDefaultPrettyPrinter();
@@ -92,7 +97,9 @@ public class JsonMetadata {
*/
public static Metadata fromJson(Reader reader) throws IOException {
Metadata m = null;
- try (JsonParser jParser = new JsonFactory().createParser(new CloseShieldReader(reader))) {
+ try (JsonParser jParser = new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build())
+ .createParser(new CloseShieldReader(reader))) {
m = readMetadataObject(jParser);
}
return m;
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
deleted file mode 100644
index 9606a81d2..000000000
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.metadata.serialization;
-
-import static org.apache.tika.metadata.serialization.JsonMetadata.readMetadataObject;
-
-import java.io.IOException;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-
-import org.apache.tika.metadata.Metadata;
-
-public class JsonMetadataDeserializer extends StdDeserializer<Metadata> {
-
- public JsonMetadataDeserializer() {
- super(Metadata.class);
- }
-
- @Override
- public Metadata deserialize(JsonParser jsonParser,
- DeserializationContext deserializationContext)
- throws IOException, JsonProcessingException {
- return readMetadataObject(jsonParser);
- }
-}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java
index e008e0564..88c21588d 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java
@@ -26,9 +26,11 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.StreamReadConstraints;
import org.apache.commons.io.input.CloseShieldReader;
import org.apache.commons.io.output.CloseShieldWriter;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -40,15 +42,18 @@ public class JsonMetadataList {
*
* @param metadataList list of metadata to write
* @param writer writer
- * @param prettyPrint whether or not to pretty print the output
+ * @param prettyPrint whether or not to pretty print the output
* @throws org.apache.tika.exception.TikaException if there is an IOException during writing
*/
- public static void toJson(List<Metadata> metadataList, Writer writer, boolean prettyPrint) throws IOException {
+ public static void toJson(List<Metadata> metadataList, Writer writer, boolean prettyPrint)
+ throws IOException {
if (metadataList == null) {
writer.write("null");
return;
}
- try (JsonGenerator jsonGenerator = new JsonFactory()
+ try (JsonGenerator jsonGenerator = new JsonFactory().setStreamReadConstraints(
+ StreamReadConstraints.builder().maxStringLength(
+ TikaConfig.getMaxJsonStringFieldLength()).build())
.createGenerator(new CloseShieldWriter(writer))) {
if (prettyPrint) {
jsonGenerator.useDefaultPrettyPrinter();
@@ -85,7 +90,9 @@ public class JsonMetadataList {
return ms;
}
ms = new ArrayList<>();
- try (JsonParser jParser = new JsonFactory().createParser(new CloseShieldReader(reader))) {
+ try (JsonParser jParser = new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build())
+ .createParser(new CloseShieldReader(reader))) {
JsonToken token = jParser.nextToken();
if (token != JsonToken.START_ARRAY) {
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
deleted file mode 100644
index ef96eb2bd..000000000
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.metadata.serialization;
-
-import static org.apache.tika.metadata.serialization.JsonMetadata.writeMetadataObject;
-
-import java.io.IOException;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-
-import org.apache.tika.metadata.Metadata;
-
-public class JsonMetadataSerializer extends StdSerializer<Metadata> {
-
- public JsonMetadataSerializer() {
- super(Metadata.class);
- }
-
- @Override
- public void serialize(Metadata metadata, JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException {
- writeMetadataObject(metadata, jsonGenerator, false);
- }
-}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java
index ac69f18a0..a0e6c4739 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java
@@ -23,7 +23,9 @@ import java.util.Arrays;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
@@ -39,7 +41,10 @@ public class JsonStreamingSerializer implements AutoCloseable {
public void add(Metadata metadata) throws IOException {
if (!hasStartedArray) {
- jsonGenerator = new JsonFactory().createGenerator(writer);
+ jsonGenerator =
+ new JsonFactory().setStreamReadConstraints(StreamReadConstraints.builder()
+ .maxStringLength(TikaConfig.getMaxJsonStringFieldLength()).build())
+ .createGenerator(writer);
jsonGenerator.writeStartArray();
hasStartedArray = true;
}
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java
index 6c2f27b18..a7b16e9ef 100644
--- a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java
@@ -21,11 +21,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.io.InputStream;
import java.io.StringReader;
import java.io.StringWriter;
import org.junit.jupiter.api.Test;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
public class JsonMetadataTest {
@@ -109,13 +111,18 @@ public class JsonMetadataTest {
@Test
public void testLargeValues() throws Exception {
+ //TIKA-4154
+ TikaConfig tikaConfig = null;
+ try (InputStream is =
+ JsonMetadata.class.getResourceAsStream("/config/tika-config-json.xml")) {
+ tikaConfig = new TikaConfig(is);
+ }
StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 1000000; i++) {
+ for (int i = 0; i < 30000000; i++) {
sb.append("v");
}
Metadata m = new Metadata();
- m.add("large_value1", sb.toString());
- m.add("large_value2", sb.toString());
+ m.add("large_value", sb.toString());
StringWriter writer = new StringWriter();
JsonMetadata.toJson(m, writer);
Metadata deserialized = JsonMetadata.fromJson(new StringReader(writer.toString()));
diff --git a/tika-serialization/src/test/resources/config/tika-config-json.xml b/tika-serialization/src/test/resources/config/tika-config-json.xml
new file mode 100644
index 000000000..8c76bd45a
--- /dev/null
+++ b/tika-serialization/src/test/resources/config/tika-config-json.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <maxJsonStringFieldLength>50000000</maxJsonStringFieldLength>
+</properties>
diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
index a427b6e00..b4861157b 100644
--- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
+++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
@@ -27,6 +27,8 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -45,8 +47,10 @@ import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition;
import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.apache.tika.TikaTest;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.OfficeOpenXMLExtended;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -578,6 +582,31 @@ public class TikaResourceTest extends CXFTestBase {
Arrays.asList(metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY_FULL_SET)));
}
+ @Test
+ public void testLargeJson(@TempDir Path dir) throws Exception {
+ //TIKA-4154
+ TikaConfig tikaConfig = null;
+ try (InputStream is =
+ JsonMetadata.class.getResourceAsStream("/config/tika-config-json.xml")) {
+ tikaConfig = new TikaConfig(is);
+ }
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 30000000; i++) {
+ sb.append("v");
+ }
+ Path tmp = Files.createTempFile(dir, "long-json-", ".txt");
+ Files.write(tmp, sb.toString().getBytes(UTF_8));
+ Response response =
+ WebClient.create(endPoint + TIKA_PATH + "/text").accept("application/json")
+ .put(Files.newInputStream(tmp));
+ Metadata metadata = JsonMetadata.fromJson(
+ new InputStreamReader(((InputStream) response.getEntity()),
+ StandardCharsets.UTF_8));
+ String t = metadata.get(TikaCoreProperties.TIKA_CONTENT);
+ assertEquals(30000000, t.trim().length());
+ }
+
@Test
public void testJsonWriteLimitEmbedded() throws Exception {
Response response =
diff --git a/tika-server/tika-server-standard/src/test/resources/config/tika-config-json.xml b/tika-server/tika-server-standard/src/test/resources/config/tika-config-json.xml
new file mode 100644
index 000000000..8c76bd45a
--- /dev/null
+++ b/tika-server/tika-server-standard/src/test/resources/config/tika-config-json.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <maxJsonStringFieldLength>50000000</maxJsonStringFieldLength>
+</properties>