You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/02/03 17:49:55 UTC
[tika] branch main updated: TIKA-3288 (#401)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new c7e184f TIKA-3288 (#401)
c7e184f is described below
commit c7e184f83fceacd9059124bcb69fb356dd0190f7
Author: Tim Allison <ta...@apache.org>
AuthorDate: Wed Feb 3 12:49:44 2021 -0500
TIKA-3288 (#401)
TIKA-3288 -- add an async handler
---
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 8 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 18 +--
.../apache/tika/batch/fs/HandlerBuilderTest.java | 2 +-
.../apache/tika/pipes/emitter/AbstractEmitter.java | 33 ++++
.../org/apache/tika/pipes/emitter/EmitData.java | 32 ++++
.../org/apache/tika/pipes/emitter/EmitKey.java | 21 ++-
.../org/apache/tika/pipes/emitter/Emitter.java | 10 +-
.../apache/tika/pipes/emitter/EmptyEmitter.java | 7 +-
.../tika/pipes/fetchiterator/FetchEmitTuple.java | 38 +++++
.../tika/pipes/fetchiterator/FetchIterator.java | 22 +++
.../org/apache/tika/eval/app/io/ExtractReader.java | 2 -
.../resources/test-dirs/extractsA/file1.pdf.json | 2 +-
tika-eval/tika-eval-core/pom.xml | 5 +
.../tika/pipes/emitter/fs/FileSystemEmitter.java | 10 +-
.../apache/tika/pipes/emitter/s3/S3Emitter.java | 4 +-
tika-pipes/tika-emitters/tika-emitter-solr/pom.xml | 5 +
.../tika/pipes/emitter/solr/SolrEmitter.java | 128 ++++++++++-----
.../apache/tika/pipes/emitter/solr/TestBasic.java | 48 ++++--
.../pipes/fetchiterator/csv/CSVFetchIterator.java | 3 +-
.../fetchiterator/jdbc/JDBCFetchIterator.java | 2 +-
.../pipes/fetchiterator/s3/S3FetchIterator.java | 2 +-
tika-pipes/tika-httpclient-commons/pom.xml | 5 -
.../org/apache/tika/client/HttpClientUtil.java | 28 ++++
.../apache/tika/pipes/PipeIntegrationTests.java | 2 +-
tika-serialization/pom.xml | 7 +-
.../tika/metadata/serialization/JsonEmitData.java | 45 ++++++
.../metadata/serialization/JsonFetchEmitTuple.java | 150 ++++++++++++++++++
.../serialization/JsonFetchEmitTupleList.java | 68 ++++++++
.../tika/metadata/serialization/JsonMetadata.java | 128 ++++++++++-----
.../metadata/serialization/JsonMetadataBase.java | 52 ------
.../serialization/JsonMetadataDeserializer.java | 73 ---------
.../metadata/serialization/JsonMetadataList.java | 80 +++++-----
.../serialization/JsonMetadataSerializer.java | 97 ------------
.../serialization/JsonStreamingSerializer.java | 34 ++--
.../serialization/PrettyMetadataKeyComparator.java | 11 +-
.../serialization/JsonFetchEmitTupleListTest.java | 62 ++++++++
.../serialization/JsonFetchEmitTupleTest.java | 53 +++++++
.../serialization/JsonMetadataListTest.java | 20 ++-
.../metadata/serialization/JsonMetadataTest.java | 23 ++-
.../tika/server/classic/TikaDetectorsTest.java | 10 +-
.../tika/server/classic/TikaMimeTypesTest.java | 8 +-
.../tika/server/classic/TikaParsersTest.java | 9 +-
tika-server/tika-server-client/pom.xml | 10 +-
.../org/apache/tika/server/client/TikaClient.java | 55 ++-----
.../apache/tika/server/client/TikaClientCLI.java | 39 ++++-
.../apache/tika/server/client/TikaHttpClient.java | 38 +++--
.../org/apache/tika/server/core/ServerStatus.java | 2 +-
.../tika/server/core/ServerStatusWatcher.java | 2 +-
.../org/apache/tika/server/core/TikaServerCli.java | 3 +
.../apache/tika/server/core/TikaServerProcess.java | 82 ++++++++--
.../tika/server/core/resource/AsyncEmitter.java | 141 +++++++++++++++++
.../tika/server/core/resource/AsyncParser.java | 142 +++++++++++++++++
.../tika/server/core/resource/AsyncRequest.java | 19 ++-
.../tika/server/core/resource/AsyncResource.java | 167 ++++++++++++++++++++
.../tika/server/core/resource/EmitterResource.java | 113 +++++++++----
.../server/core/resource/MetadataResource.java | 2 +-
.../core/resource/RecursiveMetadataResource.java | 40 +++--
.../tika/server/core/resource/TikaDetectors.java | 10 +-
.../tika/server/core/resource/TikaMimeTypes.java | 11 +-
.../tika/server/core/resource/TikaParsers.java | 17 +-
.../tika/server/core/resource/TikaResource.java | 13 +-
.../server/core/resource/UnpackerResource.java | 2 +-
.../server/core/writer/JSONMessageBodyWriter.java | 10 +-
.../tika/server/core/writer/JSONObjWriter.java | 9 +-
.../core/writer/MetadataListMessageBodyWriter.java | 10 +-
.../apache/tika/server/core/TikaEmitterTest.java | 79 +++++-----
.../apache/tika/server/core/TikaMimeTypesTest.java | 3 -
...st.java => TikaServerAsyncIntegrationTest.java} | 170 ++++++++------------
.../core/TikaServerEmitterIntegrationTest.java | 175 +++++++++++++++------
.../server/core/TikaServerIntegrationTest.java | 84 +++++-----
.../tika/server/core/TikaServerStatusTest.java | 16 +-
71 files changed, 1931 insertions(+), 900 deletions(-)
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index 2849f1a..47ba493 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -531,11 +531,7 @@ public class TikaCLI {
}
JsonMetadataList.setPrettyPrinting(prettyPrint);
Writer writer = getOutputWriter(output, encoding);
- try {
- JsonMetadataList.toJson(handler.getMetadataList(), writer);
- } finally {
- writer.flush();
- }
+ JsonMetadataList.toJson(handler.getMetadataList(), writer);
}
private ContentHandlerFactory getContentHandlerFactory(OutputType type) {
@@ -1233,7 +1229,7 @@ public class TikaCLI {
JsonMetadata.setPrettyPrinting(prettyPrint);
JsonMetadata.toJson(metadata, writer);
writer.flush();
- } catch (TikaException e) {
+ } catch (IOException e) {
throw new SAXException(e);
}
}
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
index c7d4f52..df0bbc8 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
@@ -220,10 +220,8 @@ public class TikaCLITest {
public void testJsonMetadataPrettyPrintOutput() throws Exception {
String json = getParamOutContent("--json", "-r", resourcePrefix + "testJsonMultipleInts.html");
- assertTrue(json.contains(" \"X-TIKA:Parsed-By\": [\n" +
- " \"org.apache.tika.parser.DefaultParser\",\n" +
- " \"org.apache.tika.parser.html.HtmlParser\"\n" +
- " ],\n"));
+ assertTrue(json.contains("\"X-TIKA:Parsed-By\" : [ \"org.apache.tika.parser.DefaultParser\", " +
+ "\"org.apache.tika.parser.html.HtmlParser\" ],"));
//test legacy alphabetic sort of keys
int enc = json.indexOf("\"Content-Encoding\"");
int fb = json.indexOf("fb:admins");
@@ -478,17 +476,17 @@ public class TikaCLITest {
public void testJsonRecursiveMetadataParserMetadataOnly() throws Exception {
String content = getParamOutContent("-m", "-J", "-r", resourcePrefix+"test_recursive_embedded.docx");
assertTrue(content.contains(
- "\"extended-properties:AppVersion\": \"15.0000\","));
+ "\"extended-properties:AppVersion\" : \"15.0000\","));
assertTrue(content.contains(
- "\"extended-properties:Application\": \"Microsoft Office Word\","));
- assertTrue(content.contains("\"X-TIKA:embedded_resource_path\": \"/embed1.zip\""));
+ "\"extended-properties:Application\" : \"Microsoft Office Word\","));
+ assertTrue(content.contains("\"X-TIKA:embedded_resource_path\" : \"/embed1.zip\""));
assertFalse(content.contains("X-TIKA:content"));
}
@Test
public void testJsonRecursiveMetadataParserDefault() throws Exception {
String content = getParamOutContent("-J", "-r", resourcePrefix+"test_recursive_embedded.docx");
- assertTrue(content.contains("\"X-TIKA:content\": \"\\u003chtml xmlns\\u003d\\\"http://www.w3.org/1999/xhtml"));
+ assertTrue(content.contains("\"X-TIKA:content\" : \"<html xmlns=\\\"http://www.w3.org/1999/xhtml"));
}
@Test
@@ -502,8 +500,8 @@ public class TikaCLITest {
public void testDigestInJson() throws Exception {
String content = getParamOutContent("-J", "-r", "-t", "--digest=MD5",
resourcePrefix+"test_recursive_embedded.docx");
- assertTrue(content.contains("\"X-TIKA:digest:MD5\": \"59f626e09a8c16ab6dbc2800c685f772\","));
- assertTrue(content.contains("\"X-TIKA:digest:MD5\": \"f9627095ef86c482e61d99f0cc1cf87d\""));
+ assertTrue(content.contains("\"X-TIKA:digest:MD5\" : \"59f626e09a8c16ab6dbc2800c685f772\","));
+ assertTrue(content.contains("\"X-TIKA:digest:MD5\" : \"f9627095ef86c482e61d99f0cc1cf87d\""));
}
@Test
diff --git a/tika-batch/src/test/java/org/apache/tika/batch/fs/HandlerBuilderTest.java b/tika-batch/src/test/java/org/apache/tika/batch/fs/HandlerBuilderTest.java
index 6e3648a..82a1f2b 100644
--- a/tika-batch/src/test/java/org/apache/tika/batch/fs/HandlerBuilderTest.java
+++ b/tika-batch/src/test/java/org/apache/tika/batch/fs/HandlerBuilderTest.java
@@ -109,7 +109,7 @@ public class HandlerBuilderTest extends FSBatchTestBase {
Path outputFile = outputDir.resolve("test0.xml.json");
String resultString = readFileToString(outputFile, UTF_8);
assertTrue(resultString.contains("\"author\":\"Nikolai Lobachevsky\""));
- assertTrue(resultString.contains("tika-batch\\u0027s first test file"));
+ assertTrue(resultString.contains("tika-batch's first test file"));
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
index 1117537..a54b708 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
@@ -17,6 +17,10 @@
package org.apache.tika.pipes.emitter;
import org.apache.tika.config.Field;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.util.List;
public abstract class AbstractEmitter implements Emitter {
@@ -31,4 +35,33 @@ public abstract class AbstractEmitter implements Emitter {
public String getName() {
return name;
}
+
+ /**
+ * The default behavior is to call {@link #emit(String, List)} on each item.
+ * Some implementations, e.g. Solr/ES/vespa, can benefit from subclassing this and
+ * emitting a bunch of docs at once.
+ *
+ * @param emitData
+ * @throws IOException
+ * @throws TikaEmitterException
+ */
+ @Override
+ public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
+ for (EmitData d : emitData) {
+ emit(d.getEmitKey().getKey(), d.getMetadataList());
+ }
+ }
+
+ public static long estimateSizeInBytes(String id, List<Metadata> metadataList) {
+ long sz = 36 + id.length() * 2;
+ for (Metadata m : metadataList) {
+ for (String n : m.names()) {
+ sz += 36 + n.length() * 2;
+ for (String v : m.getValues(n)) {
+ sz += 36 + v.length() * 2;
+ }
+ }
+ }
+ return sz;
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitData.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitData.java
new file mode 100644
index 0000000..0063c6a
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitData.java
@@ -0,0 +1,32 @@
+package org.apache.tika.pipes.emitter;
+
+import org.apache.tika.metadata.Metadata;
+
+import java.util.List;
+
+public class EmitData {
+
+ private final EmitKey emitKey;
+ private final List<Metadata> metadataList;
+
+ public EmitData(EmitKey emitKey, List<Metadata> metadataList) {
+ this.emitKey = emitKey;
+ this.metadataList = metadataList;
+ }
+
+ public EmitKey getEmitKey() {
+ return emitKey;
+ }
+
+ public List<Metadata> getMetadataList() {
+ return metadataList;
+ }
+
+ @Override
+ public String toString() {
+ return "EmitData{" +
+ "emitKey=" + emitKey +
+ ", metadataList=" + metadataList +
+ '}';
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
index 47a8ee7..eb4c2c2 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmitKey.java
@@ -30,7 +30,7 @@ public class EmitKey {
return emitterName;
}
- public String getEmitKey() {
+ public String getKey() {
return emitKey;
}
@@ -41,4 +41,23 @@ public class EmitKey {
", emitterKey='" + emitKey + '\'' +
'}';
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ EmitKey emitKey1 = (EmitKey) o;
+
+ if (emitterName != null ? !emitterName.equals(emitKey1.emitterName) : emitKey1.emitterName != null)
+ return false;
+ return emitKey != null ? emitKey.equals(emitKey1.emitKey) : emitKey1.emitKey == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = emitterName != null ? emitterName.hashCode() : 0;
+ result = 31 * result + (emitKey != null ? emitKey.hashCode() : 0);
+ return result;
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
index 4f6bdae..dbea669 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/Emitter.java
@@ -27,14 +27,8 @@ public interface Emitter {
void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
+ void emit(List<EmitData> emitData) throws IOException, TikaEmitterException;
+ //TODO -- add this later for xhtml?
//void emit(String txt, Metadata metadata) throws IOException, TikaException;
- /*
- TODO we can add this later?
- void addBatch(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException;
-
- void executeBatch() throws IOException, TikaEmitterException;
-
- */
-
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
index 8c0ebda..5f83db7 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/emitter/EmptyEmitter.java
@@ -16,13 +16,13 @@
*/
package org.apache.tika.pipes.emitter;
-import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import java.io.IOException;
import java.util.List;
public class EmptyEmitter implements Emitter {
+
@Override
public String getName() {
return "empty";
@@ -32,4 +32,9 @@ public class EmptyEmitter implements Emitter {
public void emit(String emitKey, List<Metadata> metadataList) throws IOException, TikaEmitterException {
}
+
+ @Override
+ public void emit(List<EmitData> emitData) throws IOException, TikaEmitterException {
+
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
index 9023378..efab236 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchEmitTuple.java
@@ -22,14 +22,25 @@ import org.apache.tika.pipes.fetcher.FetchKey;
public class FetchEmitTuple {
+ public enum ON_PARSE_EXCEPTION {
+ SKIP,
+ EMIT
+ }
+ public static final ON_PARSE_EXCEPTION DEFAULT_ON_PARSE_EXCEPTION = ON_PARSE_EXCEPTION.EMIT;
private final FetchKey fetchKey;
private final EmitKey emitKey;
private final Metadata metadata;
+ private final ON_PARSE_EXCEPTION onParseException;
public FetchEmitTuple(FetchKey fetchKey, EmitKey emitKey, Metadata metadata) {
+ this(fetchKey, emitKey, metadata, DEFAULT_ON_PARSE_EXCEPTION);
+ }
+ public FetchEmitTuple(FetchKey fetchKey, EmitKey emitKey, Metadata metadata,
+ ON_PARSE_EXCEPTION onParseException) {
this.fetchKey = fetchKey;
this.emitKey = emitKey;
this.metadata = metadata;
+ this.onParseException = onParseException;
}
public FetchKey getFetchKey() {
@@ -44,12 +55,39 @@ public class FetchEmitTuple {
return metadata;
}
+ public ON_PARSE_EXCEPTION getOnParseException() {
+ return onParseException;
+ }
+
@Override
public String toString() {
return "FetchEmitTuple{" +
"fetchKey=" + fetchKey +
", emitKey=" + emitKey +
", metadata=" + metadata +
+ ", onParseException=" + onParseException +
'}';
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ FetchEmitTuple that = (FetchEmitTuple) o;
+
+ if (fetchKey != null ? !fetchKey.equals(that.fetchKey) : that.fetchKey != null) return false;
+ if (emitKey != null ? !emitKey.equals(that.emitKey) : that.emitKey != null) return false;
+ if (metadata != null ? !metadata.equals(that.metadata) : that.metadata != null) return false;
+ return onParseException == that.onParseException;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = fetchKey != null ? fetchKey.hashCode() : 0;
+ result = 31 * result + (emitKey != null ? emitKey.hashCode() : 0);
+ result = 31 * result + (metadata != null ? metadata.hashCode() : 0);
+ result = 31 * result + (onParseException != null ? onParseException.hashCode() : 0);
+ return result;
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
index 6b9457e..505d042 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetchiterator/FetchIterator.java
@@ -49,6 +49,8 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
private String fetcherName;
private String emitterName;
private int added = 0;
+ private FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
+
public FetchIterator() {
}
@@ -95,6 +97,26 @@ public abstract class FetchIterator implements Callable<Integer>, Initializable
this.queueSize = queueSize;
}
+ @Field
+ public void setOnParseException(String onParseException) throws TikaConfigException {
+ if ("skip".equalsIgnoreCase(onParseException)) {
+ setOnParseException(FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP);
+ } else if ("emit".equalsIgnoreCase(onParseException)) {
+ setOnParseException(FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
+ } else {
+ throw new TikaConfigException("must be either 'skip' or 'emit': "
+ + onParseException);
+ }
+ }
+
+ public void setOnParseException(FetchEmitTuple.ON_PARSE_EXCEPTION onParseException) {
+ this.onParseException = onParseException;
+ }
+
+ public FetchEmitTuple.ON_PARSE_EXCEPTION getOnParseException() {
+ return onParseException;
+ }
+
@Override
public Integer call() throws Exception {
if (queue == null || numConsumers < 0) {
diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
index 40ef751..cca4c03 100644
--- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
+++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
@@ -166,8 +166,6 @@ public class ExtractReader {
}
} catch (IOException e) {
throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION);
- } catch (TikaException e) {
- throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION);
} finally {
IOUtils.closeQuietly(reader);
IOUtils.closeQuietly(is);
diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsA/file1.pdf.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsA/file1.pdf.json
index 6ef09de..8e6ae43 100644
--- a/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsA/file1.pdf.json
+++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsA/file1.pdf.json
@@ -1,5 +1,5 @@
[{
"Content-Type":"text/plain",
"X-TIKA:content":"the quick brown fox fox fox jumped over the lazy lazy dog 1,200 120000",
- "xmpTPg:NPages":2
+ "xmpTPg:NPages":"2"
}]
\ No newline at end of file
diff --git a/tika-eval/tika-eval-core/pom.xml b/tika-eval/tika-eval-core/pom.xml
index 26c087e..664ef2d 100644
--- a/tika-eval/tika-eval-core/pom.xml
+++ b/tika-eval/tika-eval-core/pom.xml
@@ -44,6 +44,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons.codec.version}</version>
diff --git a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
index f58a21c..d4a0448 100644
--- a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
@@ -41,7 +41,7 @@ import java.util.Set;
/**
* Emitter to write to a file system.
- *
+ * <p>
* This calculates the path to write to based on the {@link #basePath}
* and the value of the {@link TikaCoreProperties#SOURCE_PATH} value.
*
@@ -95,11 +95,7 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
Files.createDirectories(output.getParent());
}
try (Writer writer = Files.newBufferedWriter(output, StandardCharsets.UTF_8)) {
- try {
- JsonMetadataList.toJson(metadataList, writer);
- } catch (TikaException e) {
- throw new TikaEmitterException("can't create json", e);
- }
+ JsonMetadataList.toJson(metadataList, writer);
}
}
@@ -111,6 +107,7 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
/**
* If you want to customize the output file's file extension.
* Do not include the "."
+ *
* @param fileExtension
*/
@Field
@@ -132,6 +129,7 @@ public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter
"'; must be one of: 'skip', 'replace', 'exception'");
}
}
+
@Override
public void emit(String path, InputStream inputStream, Metadata userMetadata) throws IOException,
TikaEmitterException {
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
index d72d8fb..cb97d20 100644
--- a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3Emitter.java
@@ -111,7 +111,7 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
try (Writer writer =
new BufferedWriter(new OutputStreamWriter(bos, StandardCharsets.UTF_8))) {
JsonMetadataList.toJson(metadataList, writer);
- } catch (TikaException e) {
+ } catch (IOException e) {
throw new TikaEmitterException("can't jsonify", e);
}
byte[] bytes = bos.toByteArray();
@@ -125,7 +125,7 @@ public class S3Emitter extends AbstractEmitter implements Initializable, StreamE
try (Writer writer = Files.newBufferedWriter(tmpPath,
StandardCharsets.UTF_8, StandardOpenOption.CREATE)) {
JsonMetadataList.toJson(metadataList, writer);
- } catch (TikaException e) {
+ } catch (IOException e) {
throw new TikaEmitterException("can't jsonify", e);
}
try (InputStream is = TikaInputStream.get(tmpPath)) {
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
index 1edfff0..e36d1c8 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
@@ -42,6 +42,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
index 5830b14..0e09e9c 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitter.java
@@ -16,9 +16,8 @@
*/
package org.apache.tika.pipes.emitter.solr;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.http.client.HttpClient;
import org.apache.tika.client.HttpClientFactory;
import org.apache.tika.client.HttpClientUtil;
@@ -28,17 +27,25 @@ import org.apache.tika.config.Initializable;
import org.apache.tika.config.InitializableProblemHandler;
import org.apache.tika.config.Param;
import org.apache.tika.pipes.emitter.AbstractEmitter;
-import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.EmitData;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.emitter.TikaEmitterException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.zip.GZIPOutputStream;
public class SolrEmitter extends AbstractEmitter implements Initializable {
@@ -48,10 +55,11 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
PARENT_CHILD,
//anything else?
}
- private static final Gson GSON = new Gson();
private static final String ATTACHMENTS = "attachments";
private static final String UPDATE_PATH = "/update";
private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
+ //one day this will be allowed or can be configured?
+ private final boolean gzipJson = false;
private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
private String url;
@@ -61,27 +69,77 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
private HttpClientFactory httpClientFactory;
private HttpClient httpClient;
+ public SolrEmitter() throws TikaConfigException {
+ httpClientFactory = new HttpClientFactory();
+ }
@Override
public void emit(String emitKey, List<Metadata> metadataList) throws IOException,
TikaEmitterException {
+
if (metadataList == null || metadataList.size() == 0) {
LOG.warn("metadataList is null or empty");
return;
}
- String json = jsonify(emitKey, metadataList);
- LOG.debug("emitting json:"+json);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ Writer writer = gzipJson ?
+ new BufferedWriter(
+ new OutputStreamWriter(
+ new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
+ new BufferedWriter(
+ new OutputStreamWriter(bos, StandardCharsets.UTF_8));
+ try (
+ JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ jsonGenerator.writeStartArray();
+ jsonify(jsonGenerator, emitKey, metadataList);
+ jsonGenerator.writeEndArray();
+ }
+ LOG.debug("emitting json ({})",
+ new String(bos.toByteArray(), StandardCharsets.UTF_8));
try {
HttpClientUtil.postJson(httpClient,
- url+UPDATE_PATH+"?commitWithin="+getCommitWithin(), json);
+ url+UPDATE_PATH+"?commitWithin="+getCommitWithin(), bos.toByteArray(), gzipJson);
} catch (TikaClientException e) {
throw new TikaEmitterException("can't post", e);
}
}
- private String jsonify(String emitKey, List<Metadata> metadataList) {
+ @Override
+ public void emit(List<EmitData> batch) throws IOException,
+ TikaEmitterException {
+ if (batch == null || batch.size() == 0) {
+ LOG.warn("batch is null or empty");
+ return;
+ }
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ Writer writer = gzipJson ?
+ new BufferedWriter(
+ new OutputStreamWriter(
+ new GZIPOutputStream(bos), StandardCharsets.UTF_8)) :
+ new BufferedWriter(
+ new OutputStreamWriter(bos, StandardCharsets.UTF_8));
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ jsonGenerator.writeStartArray();
+ for (EmitData d : batch) {
+ jsonify(jsonGenerator, d.getEmitKey().getKey(), d.getMetadataList());
+ }
+ jsonGenerator.writeEndArray();
+ }
+ LOG.debug("emitting json ({})",
+ new String(bos.toByteArray(), StandardCharsets.UTF_8));
+ try {
+ HttpClientUtil.postJson(httpClient,
+ url+UPDATE_PATH+"?commitWithin="+getCommitWithin(),
+ bos.toByteArray(), gzipJson);
+ } catch (TikaClientException e) {
+ throw new TikaEmitterException("can't post", e);
+ }
+ }
+
+ private void jsonify(JsonGenerator jsonGenerator, String emitKey, List<Metadata> metadataList) throws IOException {
metadataList.get(0).set(idField, emitKey);
- if (attachmentStrategy == AttachmentStrategy.SKIP) {
- return toJsonString(jsonify(metadataList.get(0)));
+ if (attachmentStrategy == AttachmentStrategy.SKIP ||
+ metadataList.size() == 1) {
+ jsonify(metadataList.get(0), jsonGenerator);
} else if (attachmentStrategy == AttachmentStrategy.CONCATENATE_CONTENT) {
//this only handles text for now, not xhtml
StringBuilder sb = new StringBuilder();
@@ -93,53 +151,47 @@ public class SolrEmitter extends AbstractEmitter implements Initializable {
}
Metadata parent = metadataList.get(0);
parent.set(getContentField(), sb.toString());
- return toJsonString(jsonify(parent));
+ jsonify(parent, jsonGenerator);
} else if (attachmentStrategy == AttachmentStrategy.PARENT_CHILD) {
- if (metadataList.size() == 1) {
- JsonObject obj = jsonify(metadataList.get(0));
- return toJsonString(obj);
- }
- JsonObject parent = jsonify(metadataList.get(0));
- JsonArray children = new JsonArray();
+ jsonify(metadataList.get(0), jsonGenerator, false);
+ jsonGenerator.writeArrayFieldStart(ATTACHMENTS);
+
for (int i = 1; i < metadataList.size(); i++) {
Metadata m = metadataList.get(i);
m.set(idField, UUID.randomUUID().toString());
- children.add(jsonify(m));
+ jsonify(m, jsonGenerator);
}
- parent.add(ATTACHMENTS, children);
- return toJsonString(parent);
+ jsonGenerator.writeEndArray();
+ jsonGenerator.writeEndObject();
} else {
throw new IllegalArgumentException("I don't yet support this attachment strategy: "
+ attachmentStrategy);
}
}
- private String toJsonString(JsonObject obj) {
- //wrap the document into an array
- //so that Solr correctly interprets this as
- //upload docs vs a command.
- JsonArray docs = new JsonArray();
- docs.add(obj);
- return GSON.toJson(docs);
- }
-
- private JsonObject jsonify(Metadata metadata) {
- JsonObject obj = new JsonObject();
+ private void jsonify(Metadata metadata, JsonGenerator jsonGenerator, boolean writeEndObject) throws IOException {
+ jsonGenerator.writeStartObject();
for (String n : metadata.names()) {
String[] vals = metadata.getValues(n);
if (vals.length == 0) {
continue;
} else if (vals.length == 1) {
- obj.addProperty(n, vals[0]);
+ jsonGenerator.writeStringField(n, vals[0]);
} else if (vals.length > 1) {
- JsonArray valArr = new JsonArray();
- for (int i = 0; i < vals.length; i++) {
- valArr.add(vals[i]);
+ jsonGenerator.writeArrayFieldStart(n);
+ for (String val : vals) {
+ jsonGenerator.writeString(val);
}
- obj.add(n, valArr);
+ jsonGenerator.writeEndArray();
}
}
- return obj;
+ if (writeEndObject) {
+ jsonGenerator.writeEndObject();
+ }
+ }
+
+ private void jsonify(Metadata metadata, JsonGenerator jsonGenerator) throws IOException {
+ jsonify(metadata, jsonGenerator, true);
}
/**
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
index 14a091a..069e7a2 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/TestBasic.java
@@ -18,6 +18,9 @@ package org.apache.tika.pipes.emitter.solr;
import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -36,23 +39,50 @@ public class TestBasic {
TikaConfig tikaConfig = new TikaConfig(
TestBasic.class.getResourceAsStream("/tika-config-simple-emitter.xml"));
Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr1");
+ List<Metadata> metadataList = getParentChild(tikaConfig,
+ "id1", 2);
+
+ emitter.emit("1", metadataList);
+ }
+
+ @Test
+ public void testBatch() throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(
+ TestBasic.class.getResourceAsStream("/tika-config-simple-emitter.xml"));
+ Emitter emitter = tikaConfig.getEmitterManager().getEmitter("solr2");
+ List<EmitData> emitData = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ List<Metadata> metadataList = getParentChild(tikaConfig,
+ "batch_"+i, 4);
+ emitData.add(new EmitData(
+ new EmitKey(emitter.getName(), "batch_"+i),
+ metadataList));
+ }
+ emitter.emit(emitData);
+ }
+
+ private List<Metadata> getParentChild(TikaConfig tikaConfig,
+ String id, int numChildren) throws TikaException {
List<Metadata> metadataList = new ArrayList<>();
+ MetadataFilter filter = tikaConfig.getMetadataFilter();
+
Metadata m1 = new Metadata();
+ m1.set("id", id);
m1.set(Metadata.CONTENT_LENGTH, "314159");
m1.set(TikaCoreProperties.TIKA_CONTENT, "the quick brown");
m1.set(TikaCoreProperties.TITLE, "this is the first title");
m1.add(TikaCoreProperties.CREATOR, "firstAuthor");
m1.add(TikaCoreProperties.CREATOR, "secondAuthor");
-
- Metadata m2 = new Metadata();
- m2.set(TikaCoreProperties.EMBEDDED_RESOURCE_PATH, "/path_to_this.txt");
- m2.set(TikaCoreProperties.TIKA_CONTENT, "fox jumped over the lazy");
- MetadataFilter filter = tikaConfig.getMetadataFilter();
filter.filter(m1);
- filter.filter(m2);
metadataList.add(m1);
- metadataList.add(m2);
-
- emitter.emit("1", metadataList);
+ for (int i = 1; i < numChildren; i++ ) {
+ Metadata m2 = new Metadata();
+ m2.set(TikaCoreProperties.EMBEDDED_RESOURCE_PATH, "/path_to_this.txt");
+ m2.set(TikaCoreProperties.TIKA_CONTENT, "fox jumped over the lazy " + i);
+ filter.filter(m2);
+ metadataList.add(m2);
+ }
+ return metadataList;
}
+
}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
index 12268ea..7cb0c02 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-csv/src/main/java/org/apache/tika/pipes/fetchiterator/csv/CSVFetchIterator.java
@@ -120,7 +120,8 @@ public class CSVFetchIterator extends FetchIterator implements Initializable {
Metadata metadata = loadMetadata(fetchEmitKeyIndices, headers, record);
tryToAdd(new FetchEmitTuple(
new FetchKey(fetcherName, fetchKey),
- new EmitKey(emitterName, emitKey), metadata));
+ new EmitKey(emitterName, emitKey), metadata,
+ getOnParseException()));
}
}
}
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
index 2c5ac1a..19da544 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-jdbc/src/main/java/org/apache/tika/pipes/fetchiterator/jdbc/JDBCFetchIterator.java
@@ -171,7 +171,7 @@ public class JDBCFetchIterator extends FetchIterator implements Initializable {
tryToAdd(new FetchEmitTuple(
new FetchKey(fetcherName, fetchKey),
new EmitKey(emitterName, emitKey),
- metadata));
+ metadata, getOnParseException()));
}
private String toString(ResultSet rs) throws SQLException {
diff --git a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
index 2d575e9..dd14463 100644
--- a/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
+++ b/tika-pipes/tika-fetch-iterators/tika-fetch-iterator-s3/src/main/java/org/apache/tika/pipes/fetchiterator/s3/S3FetchIterator.java
@@ -104,7 +104,7 @@ public class S3FetchIterator extends FetchIterator implements Initializable {
tryToAdd(new FetchEmitTuple(
new FetchKey(fetcherName, summary.getKey()),
new EmitKey(fetcherName, summary.getKey()),
- new Metadata()));
+ new Metadata(), getOnParseException()));
count++;
}
long elapsed = System.currentTimeMillis() - start;
diff --git a/tika-pipes/tika-httpclient-commons/pom.xml b/tika-pipes/tika-httpclient-commons/pom.xml
index ef7797a..c33d3fb 100644
--- a/tika-pipes/tika-httpclient-commons/pom.xml
+++ b/tika-pipes/tika-httpclient-commons/pom.xml
@@ -42,11 +42,6 @@
<artifactId>httpclient</artifactId>
<version>${httpcomponents.version}</version>
</dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>${gson.version}</version>
- </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
index 780179c..11ebb3c 100644
--- a/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
+++ b/tika-pipes/tika-httpclient-commons/src/main/java/org/apache/tika/client/HttpClientUtil.java
@@ -20,6 +20,7 @@ import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.InputStreamEntity;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
@@ -30,7 +31,9 @@ public class HttpClientUtil {
public static boolean postJson(HttpClient client, String url, String json) throws IOException,
TikaClientException {
HttpPost post = new HttpPost(url);
+ post.setHeader("Content-Encoding", "gzip");
ByteArrayEntity entity = new ByteArrayEntity(json.getBytes(StandardCharsets.UTF_8));
+
post.setEntity(entity);
post.setHeader("Content-Type", "application/json");
HttpResponse response = client.execute(post);
@@ -48,4 +51,29 @@ public class HttpClientUtil {
return true;
}
+ public static boolean postJson(HttpClient client, String url,
+ byte[] bytes, boolean gzipped) throws IOException,
+ TikaClientException {
+ HttpPost post = new HttpPost(url);
+ if (gzipped) {
+ post.setHeader("Content-Encoding", "gzip");
+ }
+ ByteArrayEntity entity = new ByteArrayEntity(bytes);
+
+ post.setEntity(entity);
+ post.setHeader("Content-Type", "application/json");
+ HttpResponse response = client.execute(post);
+
+
+ if (response.getStatusLine().getStatusCode() != 200) {
+ String msg = EntityUtils.toString(response.getEntity());
+ throw new TikaClientException("Bad status: " +
+ response.getStatusLine().getStatusCode() + " : "+
+ msg);
+ } else {
+ String msg = EntityUtils.toString(response.getEntity());
+ System.out.println("httputil: " + msg);
+ }
+ return true;
+ }
}
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
index 7df2bb4..3831f07 100644
--- a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/PipeIntegrationTests.java
@@ -218,7 +218,7 @@ public class PipeIntegrationTests {
userMetadata.set("project", "my-project");
try (InputStream is = fetcher.fetch(t.getFetchKey().getKey(), t.getMetadata())) {
- emitter.emit(t.getEmitKey().getEmitKey(), is, userMetadata);
+ emitter.emit(t.getEmitKey().getKey(), is, userMetadata);
}
}
}
diff --git a/tika-serialization/pom.xml b/tika-serialization/pom.xml
index bc2ff93..64e3aa2 100644
--- a/tika-serialization/pom.xml
+++ b/tika-serialization/pom.xml
@@ -48,12 +48,13 @@
<groupId>${project.groupId}</groupId>
<artifactId>tika-core</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>${gson.version}</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
</dependency>
<!-- Test dependencies -->
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
new file mode 100644
index 0000000..21b6377
--- /dev/null
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonEmitData.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.serialization;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+
+import java.io.IOException;
+import java.io.Writer;
+
+public class JsonEmitData {
+
+ public static void toJson(EmitData emitData, Writer writer) throws IOException {
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ jsonGenerator.writeStartObject();
+ EmitKey key = emitData.getEmitKey();
+ jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITTER, key.getEmitterName());
+ jsonGenerator.writeStringField(JsonFetchEmitTuple.EMITKEY, key.getKey());
+ jsonGenerator.writeFieldName("data");
+ jsonGenerator.writeStartArray();
+ for (Metadata m : emitData.getMetadataList()) {
+ JsonMetadata.writeMetadataObject(m, jsonGenerator, false);
+ }
+ jsonGenerator.writeEndArray();
+ jsonGenerator.writeEndObject();
+ }
+ }
+}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
new file mode 100644
index 0000000..55d10ef
--- /dev/null
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.serialization;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.utils.StringUtils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Locale;
+
+public class JsonFetchEmitTuple {
+
+ public static final String FETCHER = "fetcher";
+ public static final String FETCHKEY = "fetchKey";
+ public static final String EMITTER = "emitter";
+ public static final String EMITKEY = "emitKey";
+ public static final String METADATAKEY = "metadata";
+ public static final String ON_PARSE_EXCEPTION = "onParseException";
+
+
+ public static FetchEmitTuple fromJson(Reader reader) throws IOException {
+ JsonParser jParser = new JsonFactory().createParser(reader);
+ JsonToken token = jParser.nextToken();
+ if (token != JsonToken.START_OBJECT) {
+ throw new IOException("require start object, but see: " + token.name());
+ }
+ return parseFetchEmitTuple(jParser);
+ }
+
+
+ static FetchEmitTuple parseFetchEmitTuple(JsonParser jParser) throws IOException {
+ JsonToken token = jParser.nextToken();
+ if (token == JsonToken.START_OBJECT) {
+ token = jParser.nextToken();
+ }
+ String fetcherName = null;
+ String fetchKey = null;
+ String emitterName = null;
+ String emitKey = null;
+ FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = null;
+ Metadata metadata = new Metadata();
+ while (token != JsonToken.END_OBJECT) {
+ if (token != JsonToken.FIELD_NAME) {
+ throw new IOException("required field name, but see: " + token.name());
+ }
+ String name = jParser.getCurrentName();
+ if (FETCHER.equals(name)) {
+ fetcherName = getValue(jParser);
+ } else if (FETCHKEY.equals(name)) {
+ fetchKey = getValue(jParser);
+ } else if (EMITTER.equals(name)) {
+ emitterName = getValue(jParser);
+ } else if (EMITKEY.equals(name)) {
+ emitKey = getValue(jParser);
+ } else if (METADATAKEY.equals(name)) {
+ token = jParser.nextToken();
+ if (token != JsonToken.START_OBJECT) {
+ throw new IOException("required start object, but see: " + token.name());
+ }
+ metadata = JsonMetadata.readMetadataObject(jParser);
+ } else if (ON_PARSE_EXCEPTION.equals(name)) {
+ String value = getValue(jParser);
+ if ("skip".equalsIgnoreCase(value)) {
+ onParseException = FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP;
+ } else if ("emit".equalsIgnoreCase(value)) {
+ onParseException = FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
+ } else {
+ throw new IOException(ON_PARSE_EXCEPTION +
+ " must be either 'skip' or 'emit'");
+ }
+ }
+ token = jParser.nextToken();
+ }
+
+ if (onParseException == null) {
+ return new FetchEmitTuple(
+ new FetchKey(fetcherName, fetchKey),
+ new EmitKey(emitterName, emitKey), metadata
+ );
+ } else {
+ return new FetchEmitTuple(
+ new FetchKey(fetcherName, fetchKey),
+ new EmitKey(emitterName, emitKey), metadata, onParseException
+ );
+ }
+ }
+
+ private static String getValue(JsonParser jParser) throws IOException {
+ JsonToken token = jParser.nextToken();
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IOException("required value string, but see: " + token.name());
+ }
+ return jParser.getValueAsString();
+ }
+
+ public static String toJson(FetchEmitTuple t) throws IOException {
+ StringWriter writer = new StringWriter();
+ toJson(t, writer);
+ return writer.toString();
+ }
+
+ public static void toJson(FetchEmitTuple t, Writer writer) throws IOException {
+
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ writeTuple(t, jsonGenerator);
+ }
+ }
+
+ static void writeTuple(FetchEmitTuple t, JsonGenerator jsonGenerator) throws IOException {
+ jsonGenerator.writeStartObject();
+ jsonGenerator.writeStringField(FETCHER, t.getFetchKey().getFetcherName());
+ jsonGenerator.writeStringField(FETCHKEY, t.getFetchKey().getKey());
+ jsonGenerator.writeStringField(EMITTER, t.getEmitKey().getEmitterName());
+ if (!StringUtils.isBlank(t.getEmitKey().getKey())) {
+ jsonGenerator.writeStringField(EMITKEY, t.getEmitKey().getKey());
+ }
+ if (t.getMetadata().size() > 0) {
+ jsonGenerator.writeFieldName(METADATAKEY);
+ JsonMetadata.writeMetadataObject(t.getMetadata(), jsonGenerator, false);
+ }
+ jsonGenerator.writeStringField(ON_PARSE_EXCEPTION,
+ t.getOnParseException().name().toLowerCase(Locale.US));
+ jsonGenerator.writeEndObject();
+
+ }
+}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java
new file mode 100644
index 0000000..4791aee
--- /dev/null
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleList.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.serialization;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.utils.StringUtils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+
+public class JsonFetchEmitTupleList {
+
+ public static List<FetchEmitTuple> fromJson(Reader reader) throws IOException {
+ JsonParser jParser = new JsonFactory().createParser(reader);
+ JsonToken token = jParser.nextToken();
+ if (token != JsonToken.START_ARRAY) {
+ throw new IOException("require start array, but see: "+token.name());
+ }
+ List<FetchEmitTuple> list = new ArrayList<>();
+ while (token != JsonToken.END_ARRAY) {
+ list.add(JsonFetchEmitTuple.parseFetchEmitTuple(jParser));
+ token = jParser.nextToken();
+ }
+ return list;
+ }
+
+ public static String toJson(List<FetchEmitTuple> list) throws IOException {
+ StringWriter writer = new StringWriter();
+ toJson(list, writer);
+ return writer.toString();
+ }
+
+ public static void toJson(List<FetchEmitTuple> list, Writer writer) throws IOException {
+
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ jsonGenerator.writeStartArray();
+ for (FetchEmitTuple t : list) {
+ JsonFetchEmitTuple.writeTuple(t, jsonGenerator);
+ }
+ jsonGenerator.writeEndArray();
+ }
+ }
+}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java
index 50bbbe5..b33c2a9 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadata.java
@@ -1,5 +1,3 @@
-package org.apache.tika.metadata.serialization;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,72 +14,130 @@ package org.apache.tika.metadata.serialization;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.tika.metadata.serialization;
+import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
+import java.util.Arrays;
-import com.google.gson.Gson;
-import com.google.gson.JsonIOException;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-public class JsonMetadata extends JsonMetadataBase{
- private static Gson GSON;
+public class JsonMetadata {
+
+ static volatile boolean PRETTY_PRINT = false;
- static {
- GSON = defaultInit();
- }
/**
* Serializes a Metadata object to Json. This does not flush or close the writer.
- *
+ *
* @param metadata metadata to write
- * @param writer writer
+ * @param writer writer
* @throws TikaException if there is an IOException during writing
*/
- public static void toJson(Metadata metadata, Writer writer) throws TikaException {
- try {
- GSON.toJson(metadata, writer);
- } catch (JsonIOException e) {
- throw new TikaException(e.getMessage());
+ public static void toJson(Metadata metadata, Writer writer) throws IOException {
+ if (metadata == null) {
+ writer.write("null");
+ return;
+ }
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ if (PRETTY_PRINT) {
+ jsonGenerator.useDefaultPrettyPrinter();
+ }
+ writeMetadataObject(metadata, jsonGenerator, PRETTY_PRINT);
}
}
-
+
+ static void writeMetadataObject(Metadata metadata,
+ JsonGenerator jsonGenerator, boolean prettyPrint) throws IOException {
+ jsonGenerator.writeStartObject();
+ String[] names = metadata.names();
+ if (prettyPrint) {
+ Arrays.sort(names, new PrettyMetadataKeyComparator());
+ }
+ for (String n : names) {
+ String[] vals = metadata.getValues(n);
+ if (vals.length == 0) {
+ continue;
+ } else if (vals.length == 1) {
+ jsonGenerator.writeStringField(n, vals[0]);
+ } else if (vals.length > 1) {
+ jsonGenerator.writeArrayFieldStart(n);
+ for (String val : vals) {
+ jsonGenerator.writeString(val);
+ }
+ jsonGenerator.writeEndArray();
+ }
+ }
+ jsonGenerator.writeEndObject();
+ }
+
/**
* Read metadata from reader.
*
* @param reader reader to read from
* @return Metadata or null if nothing could be read from the reader
- * @throws TikaException in case of parse failure by Gson or IO failure with Reader
+ * @throws IOException in case of parse failure or IO failure with Reader
*/
- public static Metadata fromJson(Reader reader) throws TikaException {
+ public static Metadata fromJson(Reader reader) throws IOException {
Metadata m = null;
- try {
- m = GSON.fromJson(reader, Metadata.class);
- } catch (com.google.gson.JsonParseException e){
- //covers both io and parse exceptions
- throw new TikaException(e.getMessage());
+ try (JsonParser jParser = new JsonFactory().createParser(reader)) {
+ m = readMetadataObject(jParser);
}
return m;
}
/**
- * Enables setting custom configurations on Gson. Remember to register
- * a serializer and a deserializer for Metadata. This does a literal set
- * and does not add the default serializer and deserializers.
- *
- * @param gson
+ * expects that jParser has not yet started on object or
+ * for jParser to be pointing to the start object.
+ * @param jParser
+ * @return
+ * @throws IOException
*/
- public static void setGson(Gson gson) {
- GSON = gson;
+ public static Metadata readMetadataObject(JsonParser jParser) throws IOException {
+ Metadata metadata = new Metadata();
+ JsonToken token = jParser.currentToken();
+ if (token == null) {
+ token = jParser.nextToken();
+ if (token != JsonToken.START_OBJECT) {
+ throw new IOException("expected start object, but got: " + token.name());
+ }
+ token = jParser.nextToken();
+ } else if (token == JsonToken.START_OBJECT) {
+ token = jParser.nextToken();
+ }
+
+ while (token != JsonToken.END_OBJECT) {
+ token = jParser.currentToken();
+ if (token != JsonToken.FIELD_NAME) {
+ throw new IOException("expected field name, but got: "
+ + token.name());
+ }
+ String key = jParser.getCurrentName();
+ token = jParser.nextToken();
+ if (token == JsonToken.START_ARRAY) {
+ while (jParser.nextToken() != JsonToken.END_ARRAY) {
+ metadata.add(key, jParser.getText());
+ }
+ } else {
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IOException("expected string value, but found: "+token.name());
+ }
+ String value = jParser.getValueAsString();
+ metadata.set(key, value);
+ }
+ token = jParser.nextToken();
+ }
+ return metadata;
}
public static void setPrettyPrinting(boolean prettyPrint) {
- if (prettyPrint) {
- GSON = prettyInit();
- } else {
- GSON = defaultInit();
- }
+ PRETTY_PRINT = prettyPrint;
}
}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataBase.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataBase.java
deleted file mode 100644
index 90bfca8..0000000
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataBase.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.apache.tika.metadata.serialization;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.Arrays;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.apache.tika.metadata.Metadata;
-
-public class JsonMetadataBase {
-
-
- static Gson defaultInit() {
- GsonBuilder builder = new GsonBuilder();
- builder.registerTypeHierarchyAdapter(Metadata.class, new JsonMetadataSerializer());
- builder.registerTypeHierarchyAdapter(Metadata.class, new JsonMetadataDeserializer());
- return builder.create();
- }
-
- static Gson prettyInit() {
- GsonBuilder builder = new GsonBuilder();
- builder.registerTypeHierarchyAdapter(Metadata.class, new SortedJsonMetadataSerializer());
- builder.registerTypeHierarchyAdapter(Metadata.class, new JsonMetadataDeserializer());
- builder.setPrettyPrinting();
- return builder.create();
- }
-
- private static class SortedJsonMetadataSerializer extends JsonMetadataSerializer {
- @Override
- public String[] getNames(Metadata m) {
- String[] names = m.names();
- Arrays.sort(names, new PrettyMetadataKeyComparator());
- return names;
- }
- }
-}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
deleted file mode 100644
index fae0141..0000000
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataDeserializer.java
+++ /dev/null
@@ -1,73 +0,0 @@
-package org.apache.tika.metadata.serialization;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.reflect.Type;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.tika.metadata.Metadata;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-
-
-/**
- * Deserializer for Metadata
- *
- * If overriding this, remember that this is called from a static context.
- * Share state only with great caution.
- */
-public class JsonMetadataDeserializer implements JsonDeserializer<Metadata> {
-
- /**
- * Deserializes a json object (equivalent to: Map<String, String[]>)
- * into a Metadata object.
- *
- * @param element to serialize
- * @param type (ignored)
- * @param context (ignored)
- * @return Metadata
- * @throws JsonParseException if element is not able to be parsed
- */
- @Override
- public Metadata deserialize(JsonElement element, Type type,
- JsonDeserializationContext context) throws JsonParseException {
-
- final JsonObject obj = element.getAsJsonObject();
- Metadata m = new Metadata();
- for (Map.Entry<String, JsonElement> entry : obj.entrySet()){
- String key = entry.getKey();
- JsonElement v = entry.getValue();
- if (v.isJsonPrimitive()){
- m.set(key, v.getAsString());
- } else if (v.isJsonArray()){
- JsonArray vArr = v.getAsJsonArray();
- for (JsonElement valueItem : vArr) {
- m.add(key, valueItem.getAsString());
- }
-
- }
- }
- return m;
- }
-}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java
index dea6faf..85f444e 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataList.java
@@ -18,25 +18,23 @@ package org.apache.tika.metadata.serialization;
*/
+import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
-import java.lang.reflect.Type;
+import java.util.ArrayList;
import java.util.List;
-import com.google.gson.Gson;
-import com.google.gson.JsonIOException;
-import com.google.gson.reflect.TypeToken;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
-public class JsonMetadataList extends JsonMetadataBase {
-
- private final static Type listType = new TypeToken<List<Metadata>>(){}.getType();
- private static Gson GSON;
- static {
- GSON = defaultInit();
- }
+public class JsonMetadataList {
+ static volatile boolean PRETTY_PRINT = false;
/**
* Serializes a Metadata object to Json. This does not flush or close the writer.
@@ -45,11 +43,20 @@ public class JsonMetadataList extends JsonMetadataBase {
* @param writer writer
* @throws org.apache.tika.exception.TikaException if there is an IOException during writing
*/
- public static void toJson(List<Metadata> metadataList, Writer writer) throws TikaException {
- try {
- GSON.toJson(metadataList, writer);
- } catch (JsonIOException e) {
- throw new TikaException(e.getMessage());
+ public static void toJson(List<Metadata> metadataList, Writer writer) throws IOException {
+ if (metadataList == null) {
+ writer.write("null");
+ return;
+ }
+ try (JsonGenerator jsonGenerator = new JsonFactory().createGenerator(writer)) {
+ if (PRETTY_PRINT) {
+ jsonGenerator.useDefaultPrettyPrinter();
+ }
+ jsonGenerator.writeStartArray();
+ for (Metadata m : metadataList) {
+ JsonMetadata.writeMetadataObject(m, jsonGenerator, PRETTY_PRINT);
+ }
+ jsonGenerator.writeEndArray();
}
}
@@ -58,18 +65,28 @@ public class JsonMetadataList extends JsonMetadataBase {
*
* @param reader
* @return Metadata or null if nothing could be read from the reader
- * @throws org.apache.tika.exception.TikaException in case of parse failure by Gson or IO failure with Reader
+ * @throws IOException in case of parse failure or IO failure with Reader
*/
- public static List<Metadata> fromJson(Reader reader) throws TikaException {
+ public static List<Metadata> fromJson(Reader reader) throws IOException {
List<Metadata> ms = null;
if (reader == null) {
return ms;
}
- try {
- ms = GSON.fromJson(reader, listType);
- } catch (com.google.gson.JsonParseException e){
- //covers both io and parse exceptions
- throw new TikaException(e.getMessage());
+ ms = new ArrayList<>();
+ try (JsonParser jParser = new JsonFactory().createParser(reader)) {
+
+ JsonToken token = jParser.nextToken();
+ if (token != JsonToken.START_ARRAY) {
+ throw new IOException(
+ "metadata list must start with an array, but I see: "+token.name());
+ }
+ token = jParser.nextToken();
+ while (token != JsonToken.END_ARRAY) {
+ Metadata m = JsonMetadata.readMetadataObject(jParser);
+ ms.add(m);
+ token = jParser.nextToken();
+ }
+
}
if (ms == null) {
return null;
@@ -88,23 +105,8 @@ public class JsonMetadataList extends JsonMetadataBase {
return ms;
}
- /**
- * Enables setting custom configurations on Gson. Remember to register
- * a serializer and a deserializer for Metadata. This does a literal set
- * and does not add the default serializer and deserializers.
- *
- * @param gson
- */
- public static void setGson(Gson gson) {
- GSON = gson;
- }
-
public static void setPrettyPrinting(boolean prettyPrint) {
- if (prettyPrint) {
- GSON = prettyInit();
- } else {
- GSON = defaultInit();
- }
+ PRETTY_PRINT = prettyPrint;
}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
deleted file mode 100644
index 602c346..0000000
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonMetadataSerializer.java
+++ /dev/null
@@ -1,97 +0,0 @@
-package org.apache.tika.metadata.serialization;
-
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-
-import org.apache.tika.metadata.Metadata;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonNull;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-
-/**
- * Serializer for Metadata
- *
- * If overriding this, remember that this is called from a static context.
- * Share state only with great caution.
- *
- */
-public class JsonMetadataSerializer implements JsonSerializer<Metadata> {
-
-
- /**
- * Serializes a Metadata object into effectively Map<String, String[]>.
- *
- * @param metadata object to serialize
- * @param type (ignored)
- * @param context (ignored)
- * @return JsonElement with key/value(s) pairs or JsonNull if metadata is null.
- */
- @Override
- public JsonElement serialize(Metadata metadata, Type type, JsonSerializationContext context) {
- if (metadata == null){
- return JsonNull.INSTANCE;
- }
- String[] names = getNames(metadata);
- if (names == null) {
- return JsonNull.INSTANCE;
- }
-
- JsonObject root = new JsonObject();
-
- for (String n : names) {
-
- String[] vals = metadata.getValues(n);
- if (vals == null) {
- //silently skip
- continue;
- }
-
- if (vals.length == 1){
- root.addProperty(n, vals[0]);
- } else {
- JsonArray jArr = new JsonArray();
- for (String val : vals) {
- jArr.add(new JsonPrimitive(val));
- }
- root.add(n, jArr);
- }
- }
- return root;
- }
-
- /**
- * Override to get a custom sort order
- * or to filter names.
- *
- * @param metadata metadata from which to grab names
- * @return list of names in the order in which they should be serialized
- */
- protected String[] getNames(Metadata metadata) {
- String[] names = metadata.names();
- Arrays.sort(names);
- return names;
- }
-}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java
index f01df6f..fb29a5e 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonStreamingSerializer.java
@@ -16,7 +16,9 @@
*/
package org.apache.tika.metadata.serialization;
-import com.google.gson.stream.JsonWriter;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.tika.metadata.Metadata;
import java.io.IOException;
@@ -26,40 +28,28 @@ import java.util.Arrays;
public class JsonStreamingSerializer implements AutoCloseable {
- private final JsonWriter jsonWriter;
+ private final Writer writer;
+ private JsonGenerator jsonGenerator;
boolean hasStartedArray = false;
public JsonStreamingSerializer(Writer writer) {
- this.jsonWriter = new JsonWriter(writer);
+ this.writer = writer;
}
public void add(Metadata metadata) throws IOException {
if (!hasStartedArray) {
- jsonWriter.beginArray();
+ jsonGenerator = new JsonFactory().createGenerator(writer);
+ jsonGenerator.writeStartArray();
hasStartedArray = true;
}
String[] names = metadata.names();
Arrays.sort(names);
- jsonWriter.beginObject();
- for (String n : names) {
- jsonWriter.name(n);
- String[] values = metadata.getValues(n);
- if (values.length == 1) {
- jsonWriter.value(values[0]);
- } else {
- jsonWriter.beginArray();
- for (String v : values) {
- jsonWriter.value(v);
- }
- jsonWriter.endArray();
- }
- }
- jsonWriter.endObject();
+ JsonMetadata.writeMetadataObject(metadata, jsonGenerator, false);
}
@Override
public void close() throws IOException {
- jsonWriter.endArray();
- jsonWriter.flush();
- jsonWriter.close();
+ jsonGenerator.writeEndArray();
+ jsonGenerator.flush();
+ jsonGenerator.close();
}
}
diff --git a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/PrettyMetadataKeyComparator.java b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/PrettyMetadataKeyComparator.java
index 5516c1d..35e76b7 100644
--- a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/PrettyMetadataKeyComparator.java
+++ b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/PrettyMetadataKeyComparator.java
@@ -17,6 +17,8 @@ package org.apache.tika.metadata.serialization;
* limitations under the License.
*/
+import org.apache.tika.metadata.TikaCoreProperties;
+
public class PrettyMetadataKeyComparator implements java.util.Comparator<String> {
@Override
public int compare(String s1, String s2) {
@@ -26,15 +28,12 @@ public class PrettyMetadataKeyComparator implements java.util.Comparator<String>
return -1;
}
- //this is stinky. This should reference AbstractRecursiveParserWrapperHandler.TIKA_CONTENT
- //but that would require making core a dependency of serialization...
- //do we want to do that?
- if (s1.equals("tika:content")) {
- if (s2.equals("tika:content")) {
+ if (s1.equals(TikaCoreProperties.TIKA_CONTENT.getName())) {
+ if (s2.equals(TikaCoreProperties.TIKA_CONTENT.getName())) {
return 0;
}
return 2;
- } else if (s2.equals("tika:content")) {
+ } else if (s2.equals(TikaCoreProperties.TIKA_CONTENT.getName())) {
return -2;
}
//do we want to lowercase?
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleListTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleListTest.java
new file mode 100644
index 0000000..80c37f4
--- /dev/null
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleListTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.serialization;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.junit.Test;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class JsonFetchEmitTupleListTest {
+
+ @Test
+ public void testBasic() throws Exception {
+ List<FetchEmitTuple> list = new ArrayList<>();
+ for (int i = 0; i < 57; i++) {
+ list.add(getFetchEmitTuple(i));
+ }
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTupleList.toJson(list, writer);
+
+ Reader reader = new StringReader(writer.toString());
+ List<FetchEmitTuple> deserialized = JsonFetchEmitTupleList.fromJson(reader);
+ assertEquals(list, deserialized);
+ }
+
+ private FetchEmitTuple getFetchEmitTuple(int i) {
+ Metadata m = new Metadata();
+ m.add("m1", "v1-"+i);
+ m.add("m1", "v1-"+i);
+ m.add("m2", "v2-"+i);
+ m.add("m2", "v3-"+i);
+ m.add("m3", "v4-"+i);
+
+ return new FetchEmitTuple(
+ new FetchKey("fetcher-"+i, "fetchkey-"+i),
+ new EmitKey("emitter-"+i, "emitKey-"+i),
+ m);
+ }
+}
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
new file mode 100644
index 0000000..67bbbde
--- /dev/null
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.metadata.serialization;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.junit.Test;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.StringWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class JsonFetchEmitTupleTest {
+
+ @Test
+ public void testBasic() throws Exception {
+ Metadata m = new Metadata();
+ m.add("m1", "v1");
+ m.add("m1", "v1");
+ m.add("m2", "v2");
+ m.add("m2", "v3");
+ m.add("m3", "v4");
+
+ FetchEmitTuple t = new FetchEmitTuple(
+ new FetchKey("fetcher1", "fetchkey1"),
+ new EmitKey("emitter1", "emitKey1"),
+ m);
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTuple.toJson(t, writer);
+
+ Reader reader = new StringReader(writer.toString());
+ FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader);
+ assertEquals(t, deserialized);
+ }
+}
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataListTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataListTest.java
index 522807e..543bbcd 100644
--- a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataListTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataListTest.java
@@ -93,7 +93,7 @@ public class JsonMetadataListTest {
@Test
public void testPrettyPrint() throws Exception {
Metadata m1 = new Metadata();
- m1.add("tika:content", "this is the content");
+ m1.add(TikaCoreProperties.TIKA_CONTENT, "this is the content");
m1.add("zk1", "v1");
m1.add("zk1", "v2");
m1.add("zk1", "v3");
@@ -114,24 +114,22 @@ public class JsonMetadataListTest {
metadataList.add(m2);
StringWriter writer = new StringWriter();
JsonMetadataList.toJson(metadataList, writer);
- assertTrue(writer.toString().startsWith("[{\"tika:content\":\"this is the content\",\"zk1\":[\"v1\",\"v2\","));
+ assertTrue(writer.toString().startsWith("["));
writer = new StringWriter();
JsonMetadataList.setPrettyPrinting(true);
JsonMetadataList.toJson(metadataList, writer);
- assertTrue(writer.toString().startsWith("[\n" +
- " {\n" +
- " \"zk1\": [\n" +
- " \"v1\",\n" +
- " \"v2\","));
- assertTrue(writer.toString().contains(" \"zk2\": \"v1\",\n" +
- " \"tika:content\": \"this is the content\"\n" +
- " },"));
+ assertTrue(writer.toString().startsWith("[ {\n" +
+ " \"zk1\" : [ \"v1\", \"v2\", \"v3\", \"v4\", \"v4\" ],\n" +
+ " \"zk2\" : \"v1\",\n" +
+ " \"X-TIKA:content\" : \"this is the content\"\n" +
+ "},"));
+
//now set it back to false
JsonMetadataList.setPrettyPrinting(false);
writer = new StringWriter();
JsonMetadataList.toJson(metadataList, writer);
- assertTrue(writer.toString().startsWith("[{\"tika:content\":\"this is the content\",\"zk1\":[\"v1\",\"v2\","));
+ assertTrue(writer.toString().startsWith("["));
}
@Test
diff --git a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java
index 4cf5367..3abeaa6 100644
--- a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java
+++ b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonMetadataTest.java
@@ -17,6 +17,7 @@ package org.apache.tika.metadata.serialization;
* limitations under the License.
*/
+import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
@@ -63,18 +64,12 @@ public class JsonMetadataTest {
JsonMetadata.setPrettyPrinting(true);
JsonMetadata.toJson(metadata, writer);
assertTrue(writer.toString().contains(
- " \"json_escapes\": \"the: \\\"quick\\\" brown, fox\",\n" +
- " \"k1\": [\n" +
- " \"v1\",\n" +
- " \"v2\"\n" +
- " ],\n" +
- " \"k3\": [\n" +
- " \"v3\",\n" +
- " \"v3\"\n" +
- " ],\n" +
- " \"k4\": \"500,000\",\n" +
- " \"url\": \"/myApp/myAction.html?method\\u003drouter\\u0026cmd\\u003d1\"\n" +
- "}"));
+ "\"json_escapes\" : \"the: \\\"quick\\\" brown, fox\",\n" +
+ " \"k1\" : [ \"v1\", \"v2\" ],\n" +
+ " \"k3\" : [ \"v3\", \"v3\" ],\n" +
+ " \"k4\" : \"500,000\",\n" +
+ " \"url\" : \"/myApp/myAction.html?method=router&cmd=1\"\n" +
+ "}"));
}
@Test
@@ -84,7 +79,7 @@ public class JsonMetadataTest {
boolean ex = false;
try {
Metadata deserialized = JsonMetadata.fromJson(new StringReader(json));
- } catch (TikaException e) {
+ } catch (IOException e) {
ex = true;
}
assertTrue(ex);
@@ -96,7 +91,7 @@ public class JsonMetadataTest {
boolean ex = false;
try {
JsonMetadata.toJson(null, writer);
- } catch (TikaException e) {
+ } catch (IOException e) {
ex = true;
}
assertFalse(ex);
diff --git a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaDetectorsTest.java b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaDetectorsTest.java
index 8d78ba5..dd5fd5c 100644
--- a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaDetectorsTest.java
+++ b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaDetectorsTest.java
@@ -26,8 +26,8 @@ import java.io.InputStream;
import java.util.List;
import java.util.Map;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
@@ -42,9 +42,6 @@ import org.junit.Test;
public class TikaDetectorsTest extends CXFTestBase {
- private static final Gson GSON = new GsonBuilder().create();
-
-
private static final String DETECTORS_PATH = "/detectors";
@Override
@@ -108,7 +105,8 @@ public class TikaDetectorsTest extends CXFTestBase {
.get();
String jsonStr = getStringFromInputStream((InputStream) response.getEntity());
- Map<String, Object> json = (Map<String, Object>) GSON.fromJson(jsonStr, Map.class);
+ Map<String, Object> json =
+ new ObjectMapper().readerFor(Map.class).readValue(jsonStr);
// Should have a nested structure
assertTrue(json.containsKey("name"));
diff --git a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaMimeTypesTest.java b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaMimeTypesTest.java
index 028c68c..b727176 100644
--- a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaMimeTypesTest.java
+++ b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaMimeTypesTest.java
@@ -16,8 +16,8 @@
*/
package org.apache.tika.server.classic;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
@@ -34,7 +34,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class TikaMimeTypesTest extends CXFTestBase {
- private static final Gson GSON = new GsonBuilder().create();
private static final String MIMETYPES_PATH = "/mime-types";
@@ -62,8 +61,7 @@ public class TikaMimeTypesTest extends CXFTestBase {
.get();
String jsonStr = getStringFromInputStream((InputStream) response.getEntity());
- Map<String, Map<String, Object>> json = (Map<String, Map<String, Object>>)
- GSON.fromJson(jsonStr, Map.class);
+ Map<String, Map<String, Object>> json = new ObjectMapper().readerFor(Map.class).readValue(jsonStr);
assertEquals(true, json.containsKey("text/plain"));
assertEquals(true, json.containsKey("application/xml"));
diff --git a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaParsersTest.java b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaParsersTest.java
index 9b065e5..771cca6 100644
--- a/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaParsersTest.java
+++ b/tika-server/tika-server-classic/src/test/java/org/apache/tika/server/classic/TikaParsersTest.java
@@ -26,8 +26,7 @@ import java.io.InputStream;
import java.util.List;
import java.util.Map;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
@@ -40,8 +39,6 @@ import org.junit.Test;
public class TikaParsersTest extends CXFTestBase {
- private static final Gson GSON = new GsonBuilder().create();
-
private static final String PARSERS_SUMMARY_PATH = "/parsers";
private static final String PARSERS_DETAILS_PATH = "/parsers/details";
@@ -142,8 +139,8 @@ public class TikaParsersTest extends CXFTestBase {
.get();
String jsonStr = getStringFromInputStream((InputStream) response.getEntity());
- Map<String, Map<String, Object>> json = (Map<String, Map<String, Object>>)
- GSON.fromJson(jsonStr, Map.class);
+ Map<String, Map<String, Object>> json =
+ new ObjectMapper().readerFor(Map.class).readValue(jsonStr);
// Should have a nested structure
assertEquals(true, json.containsKey("name"));
diff --git a/tika-server/tika-server-client/pom.xml b/tika-server/tika-server-client/pom.xml
index d3daf68..920b644 100644
--- a/tika-server/tika-server-client/pom.xml
+++ b/tika-server/tika-server-client/pom.xml
@@ -32,16 +32,16 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>tika-serialization</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponents.version}</version>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>${gson.version}</version>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
index 6e0e26e..336cb7d 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClient.java
@@ -16,27 +16,21 @@
*/
package org.apache.tika.server.client;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonPrimitive;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTupleList;
import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import java.io.IOException;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class TikaClient {
- private static final Gson GSON = new GsonBuilder().create();
-
private final Random random = new Random();
- private final TikaConfig tikaConfig;
private List<TikaHttpClient> clients;
@@ -45,11 +39,11 @@ public class TikaClient {
for (String url : tikaServers) {
clients.add(TikaHttpClient.get(url));
}
- return new TikaClient(tikaConfig, clients);
+ return new TikaClient(clients);
}
- private TikaClient(TikaConfig tikaConfig, List<TikaHttpClient> clients) {
- this.tikaConfig = tikaConfig;
+ private TikaClient(List<TikaHttpClient> clients) {
+
this.clients = clients;
}
@@ -57,37 +51,20 @@ public class TikaClient {
}*/
- public TikaEmitterResult parse(FetchEmitTuple fetchEmit, Metadata metadata)
+ public TikaEmitterResult parse(FetchEmitTuple fetchEmit)
throws IOException, TikaException {
TikaHttpClient client = getHttpClient();
- String jsonRequest = jsonifyRequest(fetchEmit, metadata);
- return client.postJson(jsonRequest);
-
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTuple.toJson(fetchEmit, writer);
+ return client.postJson(writer.toString());
}
- private String jsonifyRequest(FetchEmitTuple fetchEmit, Metadata metadata) {
- JsonObject root = new JsonObject();
- root.add("fetcher", new JsonPrimitive(fetchEmit.getFetchKey().getFetcherName()));
- root.add("fetchKey", new JsonPrimitive(fetchEmit.getFetchKey().getKey()));
- root.add("emitter", new JsonPrimitive(fetchEmit.getEmitKey().getEmitterName()));
- root.add("emitKey", new JsonPrimitive(fetchEmit.getEmitKey().getEmitKey()));
- if (metadata.size() > 0) {
- JsonObject m = new JsonObject();
- for (String n : metadata.names()) {
- String[] vals = metadata.getValues(n);
- if (vals.length == 1) {
- m.add(n, new JsonPrimitive(vals[0]));
- } else if (vals.length > 1) {
- JsonArray arr = new JsonArray();
- for (int i = 0; i < vals.length; i++) {
- arr.add(vals[i]);
- }
- m.add(n, arr);
- }
- }
- root.add("metadata", m);
- }
- return GSON.toJson(root);
+ public TikaEmitterResult parseAsync(List<FetchEmitTuple> tuples)
+ throws IOException, TikaException {
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTupleList.toJson(tuples, writer);
+ TikaHttpClient client = getHttpClient();
+ return client.postJsonAsync(writer.toString());
}
private TikaHttpClient getHttpClient() {
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
index 4b438e2..d760ab9 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaClientCLI.java
@@ -27,6 +27,7 @@ import org.xml.sax.SAXException;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -108,6 +109,42 @@ public class TikaClientCLI {
servers, numThreads);
}
+ private class AsyncFetchWorker implements Callable<Integer> {
+ private final ArrayBlockingQueue<FetchEmitTuple> queue;
+ private final TikaClient client;
+ public AsyncFetchWorker(ArrayBlockingQueue<FetchEmitTuple> queue, TikaClient client) {
+ this.queue = queue;
+ this.client = client;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ List<FetchEmitTuple> localCache = new ArrayList<>();
+ while (true) {
+
+ FetchEmitTuple t = queue.poll(maxWaitMs, TimeUnit.MILLISECONDS);
+ if (t == null) {
+ send(localCache);
+ throw new TimeoutException("exceeded maxWaitMs");
+ }
+ if (t == FetchIterator.COMPLETED_SEMAPHORE) {
+ send(localCache);
+ return 1;
+ }
+ if (localCache.size() > 20) {
+ LOGGER.debug("about to send: {}", localCache.size());
+ send(localCache);
+ localCache.clear();
+ }
+ localCache.add(t);
+ }
+ }
+
+ private void send(List<FetchEmitTuple> localCache) {
+
+ }
+ }
+
private class FetchWorker implements Callable<Integer> {
private final ArrayBlockingQueue<FetchEmitTuple> queue;
private final TikaClient client;
@@ -130,7 +167,7 @@ public class TikaClientCLI {
}
try {
LOGGER.debug("about to parse: {}", t.getFetchKey());
- client.parse(t, t.getMetadata());
+ client.parse(t);
} catch (IOException e) {
LOGGER.warn(t.getFetchKey().toString(), e);
} catch (TikaException e) {
diff --git a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
index 621f5db..591064a 100644
--- a/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
+++ b/tika-server/tika-server-client/src/main/java/org/apache/tika/server/client/TikaHttpClient.java
@@ -18,7 +18,6 @@ package org.apache.tika.server.client;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
-import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
@@ -26,6 +25,7 @@ import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,12 +39,14 @@ import java.nio.charset.StandardCharsets;
*/
class TikaHttpClient {
- private static final String ENDPOINT = "emit";
+ private static final String EMIT_ENDPOINT = "emit";
private static final String TIKA_ENDPOINT = "tika";
+ private static final String ASYNC_ENDPOINT = "async";
private static final Logger LOGGER = LoggerFactory.getLogger(TikaHttpClient.class);
private final HttpHost httpHost;
private final HttpClient httpClient;
- private final String endPointUrl;
+ private final String emitEndPointUrl;
+ private final String asyncEndPointUrl;
private final String tikaUrl;
private int maxRetries = 3;
//if can't make contact with Tika server, max wait time in ms
@@ -53,38 +55,46 @@ class TikaHttpClient {
private long pulseWaitForTikaMs = 1000;
static TikaHttpClient get(String baseUrl) throws TikaClientConfigException {
- String endPointUrl = baseUrl.endsWith("/") ? baseUrl+ENDPOINT : baseUrl+"/"+ENDPOINT;
- String tikaUrl = baseUrl.endsWith("/") ? baseUrl+TIKA_ENDPOINT : baseUrl+"/"+TIKA_ENDPOINT;
URI uri;
try {
- uri = new URI(endPointUrl);
+ uri = new URI(baseUrl);
} catch (URISyntaxException e) {
throw new TikaClientConfigException("bad URI", e);
}
HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
//TODO: need to add other configuration stuff? proxy, username, password, timeouts...
HttpClient client = HttpClients.createDefault();
- return new TikaHttpClient(endPointUrl, tikaUrl, httpHost, client);
+ return new TikaHttpClient(baseUrl, httpHost, client);
}
/**
*
- * @param endPointUrl full url to the tika-server including endpoint
- * @param tikaUrl url to /tika endpoint to use to check on server status
+ * @param baseUrl url to base endpoint
* @param httpHost
* @param httpClient
*/
- private TikaHttpClient(String endPointUrl, String tikaUrl, HttpHost httpHost, HttpClient httpClient) {
- this.endPointUrl = endPointUrl;
- this.tikaUrl = tikaUrl;
+ private TikaHttpClient(String baseUrl, HttpHost httpHost, HttpClient httpClient) {
+ if (! baseUrl.endsWith("/")) {
+ baseUrl += "/";
+ }
+ this.emitEndPointUrl = baseUrl+EMIT_ENDPOINT;
+ this.asyncEndPointUrl = baseUrl+ASYNC_ENDPOINT;
+ this.tikaUrl = baseUrl+TIKA_ENDPOINT;
this.httpHost = httpHost;
this.httpClient = httpClient;
}
+ public TikaEmitterResult postJsonAsync(String jsonRequest) {
+ return postJson(asyncEndPointUrl, jsonRequest);
+ }
public TikaEmitterResult postJson(String jsonRequest) {
- System.out.println("NED:"+endPointUrl);
- HttpPost post = new HttpPost(endPointUrl);
+ return postJson(emitEndPointUrl, jsonRequest);
+ }
+
+ private TikaEmitterResult postJson(String endPoint, String jsonRequest) {
+ System.out.println("NED:"+endPoint);
+ HttpPost post = new HttpPost(endPoint);
ByteArrayEntity entity = new ByteArrayEntity(jsonRequest.getBytes(StandardCharsets.UTF_8));
post.setEntity(entity);
post.setHeader("Content-Type", "application/json");
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java
index a15f7a5..92068f5 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatus.java
@@ -41,7 +41,7 @@ public class ServerStatus {
public enum STATUS {
INITIALIZING(0),
OPERATING(1),
- HIT_MAX(2),
+ HIT_MAX_FILES(2),
TIMEOUT(3),
ERROR(4),
PARENT_REQUESTED_SHUTDOWN(5),
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
index a2ad313..7ac0ffd 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
@@ -136,7 +136,7 @@ public class ServerStatusWatcher implements Runnable {
}
long filesProcessed = serverStatus.getFilesProcessed();
if (filesProcessed >= maxFiles) {
- serverStatus.setStatus(ServerStatus.STATUS.HIT_MAX);
+ serverStatus.setStatus(ServerStatus.STATUS.HIT_MAX_FILES);
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
index dbcbeb1..14a2ff4 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
@@ -140,6 +140,7 @@ public class TikaServerCli {
try {
mainLoop(line, newArgs);
} catch (InterruptedException e) {
+ e.printStackTrace();
//swallow
}
}
@@ -173,11 +174,13 @@ public class TikaServerCli {
WatchDogResult result = future.get();
LOG.debug("main loop future: ({}); about to restart", result);
if (maxRestarts < 0 || result.getNumRestarts() < maxRestarts) {
+ System.err.println("starting up again");
executorCompletionService.submit(
new TikaServerWatchDog(args, result.getPort(),
result.getId(),
result.getNumRestarts(), serverTimeoutConfig));
} else {
+ System.err.println("finished!");
LOG.warn("id {} with port {} has exceeded maxRestarts {}. Shutting down and not restarting.",
result.getId(), result.getPort(), maxRestarts);
finished++;
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
index 0a8fd91..5135f55 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
@@ -23,6 +23,7 @@ import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.cxf.binding.BindingFactoryManager;
+import org.apache.cxf.endpoint.Server;
import org.apache.cxf.jaxrs.JAXRSBindingFactory;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
@@ -36,6 +37,11 @@ import org.apache.tika.config.TikaConfig;
import org.apache.tika.parser.DigestingParser;
import org.apache.tika.parser.digestutils.BouncyCastleDigester;
import org.apache.tika.parser.digestutils.CommonsDigester;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.server.core.resource.AsyncEmitter;
+import org.apache.tika.server.core.resource.AsyncParser;
+import org.apache.tika.server.core.resource.AsyncResource;
import org.apache.tika.server.core.resource.DetectorResource;
import org.apache.tika.server.core.resource.EmitterResource;
import org.apache.tika.server.core.resource.LanguageResource;
@@ -72,13 +78,20 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
public class TikaServerProcess {
//used in fork mode -- restart after processing this many files
private static final long DEFAULT_MAX_FILES = 100000;
-
+ private static String ENABLE_UNSECURE_FEATURES = "enableUnsecureFeatures";
private static final int DEFAULT_DIGEST_MARK_LIMIT = 20 * 1024 * 1024;
public static final Set<String> LOG_LEVELS = new HashSet<>(Arrays.asList("debug", "info"));
@@ -129,10 +142,9 @@ public class TikaServerProcess {
LOG.info("Starting {} server", new Tika());
try {
Options options = getOptions();
-
CommandLineParser cliParser = new DefaultParser();
CommandLine line = cliParser.parse(options, args);
- runServer(line, options);
+ mainLoop(line, options);
} catch (Exception e) {
e.printStackTrace();
LOG.error("Can't start: ", e);
@@ -140,11 +152,44 @@ public class TikaServerProcess {
}
}
- //This starts the server in this process. This can
- //be either a direct call from -noFork or the process that is forked
- //in the 2.0 default mode.
- private static void runServer(CommandLine line, Options options) throws Exception {
+ private static void mainLoop(CommandLine commandLine, Options options) throws Exception {
+ AsyncResource asyncResource = null;
+ ArrayBlockingQueue<FetchEmitTuple> asyncFetchEmitQueue = null;
+ ArrayBlockingQueue<EmitData> asyncEmitData = null;
+ int numAsyncParserThreads = 10;
+ if (commandLine.hasOption(ENABLE_UNSECURE_FEATURES)) {
+ asyncResource = new AsyncResource();
+ asyncFetchEmitQueue = asyncResource.getFetchEmitQueue(10000);
+ asyncEmitData = asyncResource.getEmitDataQueue(1000);
+ }
+
+ ServerDetails serverDetails = initServer(commandLine, asyncResource);
+ ExecutorService executorService = Executors.newFixedThreadPool(numAsyncParserThreads+1);
+ ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(executorService);
+
+ if (asyncFetchEmitQueue != null) {
+ executorCompletionService.submit(new AsyncEmitter(asyncEmitData));
+ for (int i = 0; i < numAsyncParserThreads; i++) {
+ executorCompletionService.submit(new AsyncParser(asyncFetchEmitQueue, asyncEmitData));
+ }
+ }
+ //start the server
+ Server server = serverDetails.sf.create();
+ LOG.info("Started Apache Tika server {} at {}",
+ serverDetails.serverId,
+ serverDetails.url);
+
+ while (true) {
+ Future<Integer> future = executorCompletionService.poll(1, TimeUnit.MINUTES);
+ if (future != null) {
+ LOG.warn("future val: " + future.get());
+ }
+ }
+ }
+ //This returns the server, configured and ready to be started.
+ private static ServerDetails initServer(CommandLine line,
+ AsyncResource asyncResource) throws Exception {
String host = null;
if (line.hasOption("host")) {
@@ -224,12 +269,12 @@ public class TikaServerProcess {
}
InputStreamFactory inputStreamFactory = null;
- if (line.hasOption("enableUnsecureFeatures")) {
+ if (line.hasOption(ENABLE_UNSECURE_FEATURES)) {
inputStreamFactory = new FetcherStreamFactory(tika.getFetcherManager());
} else {
inputStreamFactory = new DefaultInputStreamFactory();
}
- logFetchersAndEmitters(line.hasOption("enableUnsecureFeatures"), tika);
+ logFetchersAndEmitters(line.hasOption(ENABLE_UNSECURE_FEATURES), tika);
String serverId = line.hasOption("i") ? line.getOptionValue("i") : UUID.randomUUID().toString();
LOG.debug("SERVER ID:" + serverId);
ServerStatus serverStatus;
@@ -272,8 +317,9 @@ public class TikaServerProcess {
rCoreProviders.add(new SingletonResourceProvider(new TikaDetectors()));
rCoreProviders.add(new SingletonResourceProvider(new TikaParsers()));
rCoreProviders.add(new SingletonResourceProvider(new TikaVersion()));
- if (line.hasOption("enableUnsecureFeatures")) {
+ if (line.hasOption(ENABLE_UNSECURE_FEATURES)) {
rCoreProviders.add(new SingletonResourceProvider(new EmitterResource()));
+ rCoreProviders.add(new SingletonResourceProvider(asyncResource));
}
rCoreProviders.addAll(loadResourceServices());
if (line.hasOption("status")) {
@@ -315,11 +361,11 @@ public class TikaServerProcess {
JAXRSBindingFactory factory = new JAXRSBindingFactory();
factory.setBus(sf.getBus());
manager.registerBindingFactory(JAXRSBindingFactory.JAXRS_BINDING_ID, factory);
- sf.create();
- LOG.info("Started Apache Tika server {} at {}",
- serverId,
- url);
-
+ ServerDetails details = new ServerDetails();
+ details.sf = sf;
+ details.url = url;
+ details.serverId = serverId;
+ return details;
}
private static void logFetchersAndEmitters(boolean enableUnsecureFeatures, TikaConfig tika) {
@@ -421,4 +467,10 @@ public class TikaServerProcess {
return serverTimeouts;
}
+
+ private static class ServerDetails {
+ JAXRSServerFactoryBean sf;
+ String serverId;
+ String url;
+ }
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
new file mode 100644
index 0000000..315927a
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncEmitter.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core.resource;
+
+import org.apache.tika.pipes.emitter.AbstractEmitter;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.Emitter;
+import org.apache.tika.pipes.emitter.TikaEmitterException;
+import org.apache.tika.utils.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Worker thread that takes EmitData off the queue, batches it
+ * and tries to emit it as a batch
+ */
+public class AsyncEmitter implements Callable<Integer> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncEmitter.class);
+
+ //TODO -- need to configure these
+ private final long emitWithinMs = 1000;
+
+ private long maxEstimatedBytes = 10_000_000;
+
+ private final ArrayBlockingQueue<EmitData> emitDataQueue;
+
+ Instant lastEmitted = Instant.now();
+
+ public AsyncEmitter(ArrayBlockingQueue<EmitData> emitData) {
+ this.emitDataQueue = emitData;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ EmitDataCache cache = new EmitDataCache(maxEstimatedBytes);
+
+ while (true) {
+ EmitData emitData = emitDataQueue.poll(100, TimeUnit.MILLISECONDS);
+ if (emitData != null) {
+ //this can block on emitAll
+ cache.add(emitData);
+ } else {
+ LOG.trace("Nothing on the async queue");
+ }
+ LOG.debug("cache size: ({}) bytes and count: {}",
+ cache.estimatedSize, cache.size);
+ long elapsed = ChronoUnit.MILLIS.between(lastEmitted, Instant.now());
+ if (elapsed > emitWithinMs) {
+ LOG.debug("{} elapsed > {}, going to emitAll",
+ elapsed, emitWithinMs);
+ //this can block
+ cache.emitAll();
+ }
+ }
+ }
+
+ private class EmitDataCache {
+ private final long maxBytes;
+
+ long estimatedSize = 0;
+ int size = 0;
+ Map<String, List<EmitData>> map = new HashMap<>();
+
+ public EmitDataCache(long maxBytes) {
+ this.maxBytes = maxBytes;
+ }
+
+ void updateEstimatedSize(long newBytes) {
+ estimatedSize += newBytes;
+ }
+
+ void add(EmitData data) {
+ size++;
+ long sz = AbstractEmitter.estimateSizeInBytes(data.getEmitKey().getKey(), data.getMetadataList());
+ if (estimatedSize + sz > maxBytes) {
+ LOG.debug("estimated size ({}) > maxBytes({}), going to emitAll",
+ (estimatedSize+sz), maxBytes);
+ emitAll();
+ }
+ List<EmitData> cached = map.get(data.getEmitKey().getEmitterName());
+ if (cached == null) {
+ cached = new ArrayList<>();
+ map.put(data.getEmitKey().getEmitterName(), cached);
+ }
+ updateEstimatedSize(sz);
+ cached.add(data);
+ }
+
+ private void emitAll() {
+ int emitted = 0;
+ LOG.debug("about to emit {}", size);
+ for (Map.Entry<String, List<EmitData>> e : map.entrySet()) {
+ Emitter emitter = TikaResource.getConfig()
+ .getEmitterManager().getEmitter(e.getKey());
+ tryToEmit(emitter, e.getValue());
+ emitted += e.getValue().size();
+ }
+ LOG.debug("emitted: {}", emitted);
+ estimatedSize = 0;
+ size = 0;
+ map.clear();
+ lastEmitted = Instant.now();
+ }
+
+ private void tryToEmit(Emitter emitter, List<EmitData> cachedEmitData) {
+
+ try {
+ emitter.emit(cachedEmitData);
+ } catch (IOException|TikaEmitterException e) {
+ LOG.warn("emitter class ({}): {}", emitter.getClass(),
+ ExceptionUtils.getStackTrace(e));
+ }
+ }
+ }
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
new file mode 100644
index 0000000..c20fc75
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncParser.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core.resource;
+
+import org.apache.cxf.jaxrs.impl.UriInfoImpl;
+import org.apache.cxf.message.MessageImpl;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MultivaluedHashMap;
+import java.io.InputStream;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Worker thread that takes {@link FetchEmitTuple} off the queue, parses
+ * the file and puts the {@link EmitData} on the emitDataQueue for the {@link AsyncEmitter}.
+ *
+ */
+public class AsyncParser implements Callable<Integer> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncParser.class);
+
+ private final ArrayBlockingQueue<FetchEmitTuple> fetchEmitTupleQueue;
+ private final ArrayBlockingQueue<EmitData> emitDataQueue;
+
+ public AsyncParser(ArrayBlockingQueue<FetchEmitTuple> queue,
+ ArrayBlockingQueue<EmitData> emitData) {
+ this.fetchEmitTupleQueue = queue;
+ this.emitDataQueue = emitData;
+ }
+
+ @Override
+ public Integer call() throws Exception {
+ while (true) {
+ FetchEmitTuple request = fetchEmitTupleQueue.poll(1, TimeUnit.MINUTES);
+ if (request != null) {
+ EmitData emitData = processTuple(request);
+ boolean shouldEmit = checkForParseException(request, emitData);
+ if (shouldEmit) {
+ boolean offered = emitDataQueue.offer(emitData, 10, TimeUnit.MINUTES);
+ if (!offered) {
+ //TODO: deal with this
+ LOG.warn("Failed to add ({}) " +
+ "to emit queue after 10 minutes.",
+ request.getFetchKey().getKey());
+ }
+ }
+ } else {
+ LOG.trace("Nothing on the async queue");
+ }
+ }
+ }
+
+ private boolean checkForParseException(FetchEmitTuple request, EmitData emitData) {
+ if (emitData == null || emitData.getMetadataList() == null ||
+ emitData.getMetadataList().size() == 0) {
+ LOG.warn("empty or null emit data ({})", request.getFetchKey().getKey());
+ return false;
+ }
+ boolean shouldEmit = true;
+ Metadata container = emitData.getMetadataList().get(0);
+ String stack = container.get(TikaCoreProperties.CONTAINER_EXCEPTION);
+ if (stack != null) {
+ LOG.warn("fetchKey ({}) container parse exception ({})",
+ request.getFetchKey().getKey(), stack);
+ if (request.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
+ shouldEmit = false;
+ }
+ }
+
+ for (int i = 1; i < emitData.getMetadataList().size(); i++) {
+ Metadata m = emitData.getMetadataList().get(i);
+ String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
+ if (embeddedStack != null) {
+ LOG.warn("fetchKey ({}) embedded parse exception ({})",
+ request.getFetchKey().getKey(), embeddedStack);
+ }
+ }
+ return shouldEmit;
+ }
+
+ private EmitData processTuple(FetchEmitTuple t) {
+ Metadata userMetadata = t.getMetadata();
+ Metadata metadata = new Metadata();
+ String fetcherName = t.getFetchKey().getFetcherName();
+ String fetchKey = t.getFetchKey().getKey();
+ List<Metadata> metadataList = null;
+ try (InputStream stream =
+ TikaResource.getConfig().getFetcherManager()
+ .getFetcher(fetcherName).fetch(fetchKey, metadata)) {
+ metadataList = RecursiveMetadataResource.parseMetadata(
+ stream,
+ metadata,
+ new MultivaluedHashMap<>(),
+ new UriInfoImpl(new MessageImpl()), "text");
+ } catch (SecurityException e) {
+ throw e;
+ } catch (Exception e) {
+ LOG.warn(t.toString(), e);
+ }
+
+ injectUserMetadata(userMetadata, metadataList);
+ EmitKey emitKey = t.getEmitKey();
+ if (StringUtils.isBlank(emitKey.getKey())) {
+ emitKey = new EmitKey(emitKey.getEmitterName(), fetchKey);
+ }
+ return new EmitData(emitKey, metadataList);
+ }
+
+ private void injectUserMetadata(Metadata userMetadata, List<Metadata> metadataList) {
+ for (String n : userMetadata.names()) {
+ //overwrite whatever was there
+ metadataList.get(0).set(n, null);
+ for (String val : userMetadata.getValues(n)) {
+ metadataList.get(0).add(n, val);
+ }
+ }
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncRequest.java
similarity index 69%
copy from tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
copy to tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncRequest.java
index 1117537..9e3566c 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/emitter/AbstractEmitter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncRequest.java
@@ -14,21 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.emitter;
+package org.apache.tika.server.core.resource;
-import org.apache.tika.config.Field;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
-public abstract class AbstractEmitter implements Emitter {
+import java.util.List;
- private String name;
+public class AsyncRequest {
+ private final List<FetchEmitTuple> tuples;
- @Field
- public void setName(String name) {
- this.name = name;
+ public AsyncRequest(List<FetchEmitTuple> tuples) {
+ this.tuples = tuples;
}
- @Override
- public String getName() {
- return name;
+ public List<FetchEmitTuple> getTuples() {
+ return tuples;
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
new file mode 100644
index 0000000..248ebb0
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/AsyncResource.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.server.core.resource;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTupleList;
+import org.apache.tika.pipes.emitter.EmitData;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.emitter.EmitterManager;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.UriInfo;
+import java.io.IOException;
+import java.io.InputStream;
+
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+@Path("/async")
+public class AsyncResource {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncResource.class);
+
+ private static final int DEFAULT_FETCH_EMIT_QUEUE_SIZE = 10000;
+ long maxQueuePauseMs = 60000;
+ private ArrayBlockingQueue<FetchEmitTuple> queue;
+
+ public ArrayBlockingQueue<FetchEmitTuple> getFetchEmitQueue(int queueSize) {
+ this.queue = new ArrayBlockingQueue<>(queueSize);
+ return queue;
+ }
+
+ public ArrayBlockingQueue<EmitData> getEmitDataQueue(int size) {
+ return new ArrayBlockingQueue<>(size);
+ }
+ /**
+ * The client posts a json request. At a minimum, this must be a
+ * json object that contains an emitter and a fetcherString key with
+ * the key to fetch the inputStream. Optionally, it may contain a metadata
+ * object that will be used to populate the metadata key for pass
+ * through of metadata from the client.
+ * <p>
+ * The extracted text content is stored with the key
+ * {@link TikaCoreProperties#TIKA_CONTENT}
+ * <p>
+ * Must specify a fetcherString and an emitter in the posted json.
+ *
+ * @param info uri info
+ * @return InputStream that can be deserialized as a list of {@link Metadata} objects
+ * @throws Exception
+ */
+ @POST
+ @Produces("application/json")
+ public Map<String, Object> post(InputStream is,
+ @Context HttpHeaders httpHeaders,
+ @Context UriInfo info
+ ) throws Exception {
+
+ AsyncRequest request = deserializeASyncRequest(is);
+
+ //make sure that there are no problems with
+ //the requested fetchers and emitters
+ //throw early
+ FetcherManager fetcherManager = TikaResource.getConfig().getFetcherManager();
+ EmitterManager emitterManager = TikaResource.getConfig().getEmitterManager();
+ for (FetchEmitTuple t : request.getTuples()) {
+ if (! fetcherManager.getSupported().contains(t.getFetchKey().getFetcherName())) {
+ return badFetcher(t.getFetchKey());
+ }
+ if (! emitterManager.getSupported().contains(t.getEmitKey().getEmitterName())) {
+ return badEmitter(t.getEmitKey());
+ }
+ }
+ Instant start = Instant.now();
+ long elapsed = ChronoUnit.MILLIS.between(start, Instant.now());
+ List<FetchEmitTuple> notAdded = new ArrayList<>();
+ int addedCount = 0;
+ for (FetchEmitTuple t : request.getTuples()) {
+ boolean offered = false;
+ while (!offered && elapsed < maxQueuePauseMs) {
+ offered = queue.offer(t, 10, TimeUnit.MILLISECONDS);
+ elapsed = ChronoUnit.MILLIS.between(start, Instant.now());
+ }
+ if (! offered) {
+ notAdded.add(t);
+ } else {
+ addedCount++;
+ }
+ }
+
+ if (notAdded.size() > 0) {
+ return throttle(notAdded, addedCount);
+ }
+ return ok(request.getTuples().size());
+ }
+
+ private Map<String, Object> ok(int size) {
+ Map<String, Object> map = new HashMap<>();
+ map.put("status", "ok");
+ map.put("added", size);
+ return map;
+ }
+
+ private Map<String, Object> throttle(List<FetchEmitTuple> notAdded, int added) {
+ List<String> fetchKeys = new ArrayList<>();
+ for (FetchEmitTuple t : notAdded) {
+ fetchKeys.add(t.getFetchKey().getKey());
+ }
+ Map<String, Object> map = new HashMap<>();
+ map.put("status", "throttled");
+ map.put("added", added);
+ map.put("skipped", notAdded.size());
+ map.put("skippedFetchKeys", fetchKeys);
+ return map;
+ }
+
+ private Map<String, Object> badEmitter(EmitKey emitKey) {
+ throw new BadRequestException("can't find emitter for "+emitKey.getEmitterName());
+ }
+
+ private Map<String, Object> badFetcher(FetchKey fetchKey) {
+ throw new BadRequestException("can't find fetcher for " + fetchKey.getFetcherName());
+ }
+
+ private AsyncRequest deserializeASyncRequest(InputStream is) throws IOException {
+ try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) {
+ return new AsyncRequest(JsonFetchEmitTupleList.fromJson(reader));
+ }
+ }
+
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
index 9689ba8..6338dc4 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
@@ -17,15 +17,19 @@
package org.apache.tika.server.core.resource;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import org.apache.tika.Tika;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.pipes.emitter.TikaEmitterException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import org.apache.tika.utils.ExceptionUtils;
import org.apache.tika.utils.StringUtils;
import org.slf4j.Logger;
@@ -46,6 +50,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -95,7 +100,7 @@ public class EmitterResource {
httpHeaders.getRequestHeaders(), info, "text");
}
emitKey = StringUtils.isBlank(emitKey) ? fetchKey : emitKey;
- return emit(emitterName, emitKey, metadataList);
+ return emit(new EmitKey(emitterName, emitKey), metadataList);
}
/**
@@ -129,7 +134,7 @@ public class EmitterResource {
RecursiveMetadataResource.parseMetadata(is,
metadata,
httpHeaders.getRequestHeaders(), info, "text");
- return emit(emitterName, emitKey, metadataList);
+ return emit(new EmitKey(emitterName, emitKey), metadataList);
}
/**
@@ -154,52 +159,92 @@ public class EmitterResource {
@Context HttpHeaders httpHeaders,
@Context UriInfo info
) throws Exception {
- JsonObject root = null;
+ FetchEmitTuple t = null;
try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8)) {
- root = JsonParser.parseReader(reader).getAsJsonObject();
+ t = JsonFetchEmitTuple.fromJson(reader);
}
- String fetcherName = root.get("fetcher").getAsString();
- String fetchKey = root.get("fetchKey").getAsString();
- String emitKey = (root.has("emitKey")) ?
- root.get("emitKey").getAsString() : fetchKey;
- String emitterName = root.get("emitter").getAsString();
Metadata metadata = new Metadata();
-
List<Metadata> metadataList = null;
try (InputStream stream =
TikaResource.getConfig().getFetcherManager()
- .getFetcher(fetcherName).fetch(fetchKey, metadata)) {
+ .getFetcher(t.getFetchKey().getFetcherName())
+ .fetch(t.getFetchKey().getKey(), metadata)) {
metadataList = RecursiveMetadataResource.parseMetadata(
stream,
metadata,
httpHeaders.getRequestHeaders(), info, "text");
} catch (Error error) {
- return returnError(emitterName, error);
+ return returnError(t.getEmitKey().getEmitterName(), error);
+ }
+
+ boolean shouldEmit = checkParseException(t, metadataList);
+ if (! shouldEmit) {
+ return skip(t, metadataList);
}
- injectUserMetadata(metadataList.get(0), root.getAsJsonObject());
+ injectUserMetadata(t.getMetadata(), metadataList.get(0));
for (String n : metadataList.get(0).names()) {
LOG.debug("post parse/pre emit metadata {}: {}",
n, metadataList.get(0).get(n));
}
- return emit(emitterName, emitKey, metadataList);
+ return emit(calcEmitKey(t), metadataList);
+ }
+
+ static EmitKey calcEmitKey(FetchEmitTuple t) {
+ //use fetch key if emitter key is not specified
+ //TODO: clean this up?
+ EmitKey emitKey = t.getEmitKey();
+ if (StringUtils.isBlank(emitKey.getKey())) {
+ emitKey = new EmitKey(emitKey.getEmitterName(), t.getFetchKey().getKey());
+ }
+ return emitKey;
+ }
+
+ private Map<String, String> skip(FetchEmitTuple t, List<Metadata> metadataList) {
+ Map<String, String> statusMap = new HashMap<>();
+ statusMap.put("status", "ok");
+ statusMap.put("emitter", t.getEmitKey().getEmitterName());
+ statusMap.put("emitKey", t.getEmitKey().getKey());
+ String msg = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
+ statusMap.put("parse_exception", msg);
+ return statusMap;
}
- private void injectUserMetadata(Metadata metadata, JsonObject root) {
- if (root.has("metadata")) {
- JsonObject meta = root.getAsJsonObject("metadata");
- for (String k : meta.keySet()) {
- JsonElement val = meta.get(k);
- if (val.isJsonArray()) {
- for (JsonElement v : val.getAsJsonArray()) {
- metadata.add(k, v.getAsString());
- }
- } else {
- metadata.set(k, val.getAsString());
- }
+ private boolean checkParseException(FetchEmitTuple t, List<Metadata> metadataList) {
+ if (metadataList == null || metadataList.size() < 1) {
+ return false;
+ }
+ boolean shouldEmit = true;
+ String stack = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION);
+ if (stack != null) {
+ if (t.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
+ shouldEmit = false;
+ }
+ LOG.warn("fetchKey ({}) container parse exception ({})",
+ t.getFetchKey().getKey(), stack);
+ }
+
+ for (int i = 1; i < metadataList.size(); i++) {
+ Metadata m = metadataList.get(i);
+ String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
+ if (embeddedStack != null) {
+ LOG.warn("fetchKey ({}) embedded parse exception ({})",
+ t.getFetchKey().getKey(), embeddedStack);
+ }
+ }
+
+ return shouldEmit;
+ }
+
+
+ private void injectUserMetadata(Metadata userMetadata, Metadata metadata) {
+ for (String n : userMetadata.names()) {
+ metadata.set(n, null);
+ for (String v : userMetadata.getValues(n)) {
+ metadata.add(n, v);
}
}
}
@@ -213,20 +258,20 @@ public class EmitterResource {
return statusMap;
}
- private Map<String, String> emit(String emitterName, String emitKey, List<Metadata> metadataList) throws TikaException {
- Emitter emitter = TikaResource.getConfig().getEmitterManager().getEmitter(emitterName);
+ private Map<String, String> emit(EmitKey emitKey, List<Metadata> metadataList) throws TikaException {
+ Emitter emitter = TikaResource.getConfig().getEmitterManager().getEmitter(emitKey.getEmitterName());
String status = "ok";
String exceptionMsg = "";
try {
- emitter.emit(emitKey, metadataList);
+ emitter.emit(emitKey.getKey(), metadataList);
} catch (IOException | TikaEmitterException e) {
- LOG.warn("problem with emitting", e);
+ LOG.warn("problem emitting ("+emitKey.getKey()+")", e);
status = "emitter_exception";
exceptionMsg = ExceptionUtils.getStackTrace(e);
}
Map<String, String> statusMap = new HashMap<>();
statusMap.put("status", status);
- statusMap.put("emitter", emitterName);
+ statusMap.put("emitter", emitKey.getEmitterName());
if (exceptionMsg.length() > 0) {
statusMap.put("emitter_exception", exceptionMsg);
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
index 8668b16..7a634fd 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/MetadataResource.java
@@ -132,7 +132,7 @@ public class MetadataResource {
//no need to parse embedded docs
context.set(DocumentSelector.class, metadata1 -> false);
- TikaResource.logRequest(LOG, info, metadata);
+ TikaResource.logRequest(LOG, "/meta", metadata);
TikaResource.parse(parser, LOG, info.getPath(), is,
new LanguageHandler() {
public void endDocument() {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
index 298d848..f6f340c 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java
@@ -23,6 +23,7 @@ import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MultivaluedMap;
@@ -39,6 +40,7 @@ import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
import org.apache.tika.server.core.MetadataList;
+import org.apache.tika.server.core.TikaServerParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,7 +64,7 @@ public class RecursiveMetadataResource {
* just the immediate children.
* <p>
* The extracted text content is stored with the key
- * {@link RecursiveParserWrapper#TIKA_CONTENT}.
+ * {@link org.apache.tika.metadata.TikaCoreProperties#TIKA_CONTENT}.
* <p>
* Specify the handler for the content (xml, html, text, ignore)
* in the path:<br/>
@@ -98,7 +100,7 @@ public class RecursiveMetadataResource {
* just the immediate children.
* <p>
* The extracted text content is stored with the key
- * {@link RecursiveParserWrapper#TIKA_CONTENT}.
+ * {@link org.apache.tika.metadata.TikaCoreProperties#TIKA_CONTENT}.
* <p>
* Specify the handler for the content (xml, html, text, ignore)
* in the path:<br/>
@@ -134,15 +136,15 @@ public class RecursiveMetadataResource {
UriInfo info, String handlerTypeName)
throws Exception {
- final ParseContext context = new ParseContext();
- Parser parser = TikaResource.createParser();
- // TODO: parameterize choice of max chars/max embedded attachments
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
+ final ParseContext context = new ParseContext();
+ Parser parser = TikaResource.createParser();
+ // TODO: parameterize choice of max chars/max embedded attachments
+ RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
fillMetadata(parser, metadata, httpHeaders);
fillParseContext(httpHeaders, metadata, context);
- TikaResource.logRequest(LOG, info, metadata);
+ TikaResource.logRequest(LOG, "/rmeta", metadata);
int writeLimit = -1;
if (httpHeaders.containsKey("writeLimit")) {
@@ -156,24 +158,20 @@ public class RecursiveMetadataResource {
BasicContentHandlerFactory.HANDLER_TYPE type =
BasicContentHandlerFactory.parseHandlerType(handlerTypeName, DEFAULT_HANDLER_TYPE);
- RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
- new BasicContentHandlerFactory(type, writeLimit), maxEmbeddedResources,
+ RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
+ new BasicContentHandlerFactory(type, writeLimit), maxEmbeddedResources,
TikaResource.getConfig().getMetadataFilter());
- try {
- TikaResource.parse(wrapper, LOG, info.getPath(), is, handler, metadata, context);
- } catch (SecurityException e) {
+ try {
+ TikaResource.parse(wrapper, LOG, "/rmeta", is, handler, metadata, context);
+ } catch (TikaServerParseException e) {
+ //do nothing
+ } catch (SecurityException|WebApplicationException e) {
throw e;
} catch (Exception e) {
- //swallow it and report it via the metadata list
+ //we shouldn't get here?
+ e.printStackTrace();
}
- /*
- We used to have this non-functional bit of code...refactor to add it back and make it work?
- new LanguageHandler() {
- public void endDocument() {
- metadata.set("language", getLanguage().getLanguage());
- }
- },
- */
+
return handler.getMetadataList();
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java
index ce163d5..5b738fb 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaDetectors.java
@@ -19,13 +19,13 @@ package org.apache.tika.server.core.resource;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.tika.detect.CompositeDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.server.core.HTMLHelper;
@@ -36,8 +36,6 @@ import org.apache.tika.server.core.HTMLHelper;
*/
@Path("/detectors")
public class TikaDetectors {
- private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create();
-
private HTMLHelper html;
@@ -77,10 +75,10 @@ public class TikaDetectors {
@GET
@Produces(javax.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getDetectorsJSON() {
+ public String getDetectorsJSON() throws IOException {
Map<String, Object> details = new HashMap<String, Object>();
detectorAsMap(TikaResource.getConfig().getDetector(), details);
- return GSON.toJson(details);
+ return new ObjectMapper().writeValueAsString(details);
}
private void detectorAsMap(Detector d, Map<String, Object> details) {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java
index be1d84b..c45fb25 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaMimeTypes.java
@@ -19,6 +19,7 @@ package org.apache.tika.server.core.resource;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -26,8 +27,7 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MediaTypeRegistry;
import org.apache.tika.parser.CompositeParser;
@@ -41,8 +41,6 @@ import org.apache.tika.server.core.HTMLHelper;
@Path("/mime-types")
public class TikaMimeTypes {
- private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create();
-
private HTMLHelper html;
public TikaMimeTypes() {
@@ -94,7 +92,7 @@ public class TikaMimeTypes {
@GET
@Produces(javax.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getMimeTypesJSON() {
+ public String getMimeTypesJSON() throws IOException {
Map<String, Object> details = new HashMap<String, Object>();
for (MediaTypeDetails type : getMediaTypes()) {
@@ -111,7 +109,8 @@ public class TikaMimeTypes {
details.put(type.type.toString(), typeDets);
}
- return GSON.toJson(details);
+ return new ObjectMapper().writerWithDefaultPrettyPrinter()
+ .writeValueAsString(details);
}
private static String[] copyToStringArray(MediaType[] aliases) {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java
index f1c4abb..400230c 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaParsers.java
@@ -19,6 +19,7 @@ package org.apache.tika.server.core.resource;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -28,8 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.ParseContext;
@@ -45,7 +45,6 @@ import org.apache.tika.server.core.HTMLHelper;
@Path("/parsers")
public class TikaParsers {
private static final ParseContext EMPTY_PC = new ParseContext();
- private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create();
private HTMLHelper html;
public TikaParsers() {
@@ -116,21 +115,23 @@ public class TikaParsers {
@GET
@Path("/details")
@Produces(javax.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getParserDetailsJSON() {
+ public String getParserDetailsJSON() throws IOException {
return getParsersJSON(true);
}
@GET
@Produces(javax.ws.rs.core.MediaType.APPLICATION_JSON)
- public String getParsersJSON() {
+ public String getParsersJSON() throws IOException {
return getParsersJSON(false);
}
- protected String getParsersJSON(boolean withMimeTypes) {
+ protected String getParsersJSON(boolean withMimeTypes) throws IOException {
Map<String, Object> details = new HashMap<String, Object>();
parserAsMap(new ParserDetails(TikaResource.getConfig().getParser()), withMimeTypes, details);
-
- return GSON.toJson(details);
+ ObjectMapper objectMapper = new ObjectMapper();
+ return objectMapper
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(details);
}
private void parserAsMap(ParserDetails p, boolean withMimeTypes, Map<String, Object> details) {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index 46cf093..d2c33e9 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -354,11 +354,12 @@ public class TikaResource {
}
}
- public static void logRequest(Logger logger, UriInfo info, Metadata metadata) {
+ public static void logRequest(Logger logger, String endpoint, Metadata metadata) {
+
if (metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE) == null) {
- logger.info("{} (autodetecting type)", info.getPath());
+ logger.info("{} (autodetecting type)", endpoint);
} else {
- logger.info("{} ({})", info.getPath(), metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE));
+ logger.info("{} ({})", endpoint, metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE));
}
}
@@ -403,7 +404,7 @@ public class TikaResource {
fillMetadata(parser, metadata, httpHeaders);
fillParseContext(httpHeaders, metadata, context);
- logRequest(LOG, info, metadata);
+ logRequest(LOG, "/tika", metadata);
return new StreamingOutput() {
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
@@ -432,7 +433,7 @@ public class TikaResource {
fillMetadata(parser, metadata, httpHeaders);
fillParseContext(httpHeaders, metadata, context);
- logRequest(LOG, info, metadata);
+ logRequest(LOG, "/tika", metadata);
return new StreamingOutput() {
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
@@ -489,7 +490,7 @@ public class TikaResource {
fillParseContext(httpHeaders, metadata, context);
- logRequest(LOG, info, metadata);
+ logRequest(LOG, "/tika", metadata);
return new StreamingOutput() {
public void write(OutputStream outputStream)
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
index ed0320b..a3d733a 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
@@ -126,7 +126,7 @@ public class UnpackerResource {
fillMetadata(parser, metadata, httpHeaders.getRequestHeaders());
fillParseContext(httpHeaders.getRequestHeaders(), metadata, pc);
- TikaResource.logRequest(LOG, info, metadata);
+ TikaResource.logRequest(LOG, "/unpack", metadata);
//even though we aren't currently parsing embedded documents,
//we need to add this to allow for "inline" use of other parsers.
pc.set(Parser.class, parser);
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java
index d3ff0a8..a0438af 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONMessageBodyWriter.java
@@ -53,13 +53,9 @@ public class JSONMessageBodyWriter implements MessageBodyWriter<Metadata> {
public void writeTo(Metadata metadata, Class<?> type, Type genericType, Annotation[] annotations,
MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream) throws IOException,
WebApplicationException {
- try {
- Writer writer = new OutputStreamWriter(entityStream, UTF_8);
- JsonMetadata.toJson(metadata, writer);
- writer.flush();
- } catch (TikaException e) {
- throw new IOException(e);
- }
+ Writer writer = new OutputStreamWriter(entityStream, UTF_8);
+ JsonMetadata.toJson(metadata, writer);
+ writer.flush();
entityStream.flush();
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java
index f71fde3..c79ac48 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/JSONObjWriter.java
@@ -17,8 +17,7 @@
package org.apache.tika.server.core.writer;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.tika.metadata.Metadata;
import javax.ws.rs.Produces;
@@ -40,7 +39,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
@Provider
@Produces(MediaType.APPLICATION_JSON)
public class JSONObjWriter implements MessageBodyWriter<Map<String, Object>> {
- private static Gson GSON = new GsonBuilder().setPrettyPrinting().create();
+
public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
return Map.class.isAssignableFrom(type);
@@ -55,8 +54,8 @@ public class JSONObjWriter implements MessageBodyWriter<Map<String, Object>> {
MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream)
throws IOException, WebApplicationException {
Writer writer = new OutputStreamWriter(entityStream, UTF_8);
- GSON.toJson(map, writer);
- writer.flush();
+ ObjectMapper objectMapper = new ObjectMapper();
+ objectMapper.writerWithDefaultPrettyPrinter().writeValue(writer, map);
entityStream.flush();
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
index cedfcba..647da2d 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
@@ -56,13 +56,9 @@ public class MetadataListMessageBodyWriter implements MessageBodyWriter<Metadata
public void writeTo(MetadataList list, Class<?> type, Type genericType, Annotation[] annotations,
MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream) throws IOException,
WebApplicationException {
- try {
- Writer writer = new OutputStreamWriter(entityStream, UTF_8);
- JsonMetadataList.toJson(list.getMetadata(), writer);
- writer.flush();
- } catch (TikaException e) {
- throw new IOException(e);
- }
+ Writer writer = new OutputStreamWriter(entityStream, UTF_8);
+ JsonMetadataList.toJson(list.getMetadata(), writer);
+ writer.flush();
entityStream.flush();
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java
index 31a3cd0..3c92674 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaEmitterTest.java
@@ -16,11 +16,8 @@
*/
package org.apache.tika.server.core;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
@@ -29,7 +26,11 @@ import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import org.apache.tika.server.core.resource.EmitterResource;
import org.apache.tika.server.core.writer.JSONObjWriter;
import org.junit.AfterClass;
@@ -42,6 +43,7 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -68,6 +70,11 @@ public class TikaEmitterTest extends CXFTestBase {
private static String HELLO_WORLD = "hello_world.xml";
private static String HELLO_WORLD_JSON = "hello_world.xml.json";
+ private static String[] VALUE_ARRAY = new String[]{
+ "my-value-1",
+ "my-value-2",
+ "my-value-3"
+ };
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TMP_DIR = Files.createTempDirectory("tika-emitter-test-");
@@ -128,6 +135,7 @@ public class TikaEmitterTest extends CXFTestBase {
@Override
protected void setUpProviders(JAXRSServerFactoryBean sf) {
List<Object> providers = new ArrayList<>();
+ providers.add(new TikaServerParseExceptionMapper(true));
providers.add(new JSONObjWriter());
sf.setProviders(providers);
}
@@ -167,27 +175,24 @@ public class TikaEmitterTest extends CXFTestBase {
@Test
public void testPost() throws Exception {
- JsonObject root = new JsonObject();
- root.add("fetcher", new JsonPrimitive("fsf"));
- root.add("fetchKey", new JsonPrimitive("hello_world.xml"));
- root.add("emitter", new JsonPrimitive("fse"));
- JsonObject userMetadata = new JsonObject();
- String[] valueArray = new String[] {"my-value-1", "my-value-2", "my-value-3"};
- JsonArray arr = new JsonArray();
- for (int i = 0; i < valueArray.length; i++) {
- arr.add(valueArray[i]);
+ Metadata userMetadata = new Metadata();
+ userMetadata.set("my-key", "my-value");
+ for (int i = 0; i < VALUE_ARRAY.length; i++) {
+ userMetadata.add("my-key-multi", VALUE_ARRAY[i]);
}
- userMetadata.add("my-key", new JsonPrimitive("my-value"));
- userMetadata.add("my-key-multi", arr);
- root.add("metadata", userMetadata);
- String jsonPost = new Gson().toJson(root);
+ FetchEmitTuple t = new FetchEmitTuple(
+ new FetchKey("fsf", "hello_world.xml"),
+ new EmitKey("fse", ""),
+ userMetadata);
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTuple.toJson(t, writer);
String getUrl = endPoint+EMITTER_PATH;
Response response = WebClient
.create(getUrl)
.accept("application/json")
- .post(jsonPost);
+ .post(writer.toString());
assertEquals(200, response.getStatus());
List<Metadata> metadataList = null;
@@ -202,7 +207,7 @@ public class TikaEmitterTest extends CXFTestBase {
assertEquals("你好，世界", metadata.get("title"));
assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
assertEquals("my-value", metadata.get("my-key"));
- assertArrayEquals(valueArray, metadata.getValues("my-key-multi"));
+ assertArrayEquals(VALUE_ARRAY, metadata.getValues("my-key-multi"));
}
@Test
@@ -235,36 +240,32 @@ public class TikaEmitterTest extends CXFTestBase {
@Test
public void testPostNPE() throws Exception {
-
- JsonObject root = new JsonObject();
- root.add("fetcher", new JsonPrimitive("fsf"));
- root.add("fetchKey", new JsonPrimitive("null_pointer.xml"));
- root.add("emitter", new JsonPrimitive("fse"));
- JsonObject userMetadata = new JsonObject();
- String[] valueArray = new String[] {"my-value-1", "my-value-2", "my-value-3"};
- JsonArray arr = new JsonArray();
- for (int i = 0; i < valueArray.length; i++) {
- arr.add(valueArray[i]);
+ Metadata userMetadata = new Metadata();
+ userMetadata.set("my-key", "my-value");
+ for (int i = 0; i < VALUE_ARRAY.length; i++) {
+ userMetadata.add("my-key-multi", VALUE_ARRAY[i]);
}
- userMetadata.add("my-key", new JsonPrimitive("my-value"));
- userMetadata.add("my-key-multi", arr);
- root.add("metadata", userMetadata);
- String jsonPost = new Gson().toJson(root);
+ FetchEmitTuple t = new FetchEmitTuple(
+ new FetchKey("fsf", "null_pointer.xml"),
+ new EmitKey("fse", ""),
+ userMetadata);
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTuple.toJson(t, writer);
String getUrl = endPoint+EMITTER_PATH;
Response response = WebClient
.create(getUrl)
.accept("application/json")
- .post(jsonPost);
+ .post(writer.toString());
assertEquals(200, response.getStatus());
- JsonObject jsonResponse;
+ JsonNode jsonResponse;
try (Reader reader = new InputStreamReader(
(InputStream)response.getEntity(), StandardCharsets.UTF_8)) {
- jsonResponse = JsonParser.parseReader(reader).getAsJsonObject();
+ jsonResponse = new ObjectMapper().readTree(reader);
};
- String parseException = jsonResponse.get("parse_exception").getAsString();
+ String parseException = jsonResponse.get("parse_exception").asText();
assertNotNull(parseException);
assertContains("NullPointerException", parseException);
@@ -276,7 +277,7 @@ public class TikaEmitterTest extends CXFTestBase {
Metadata metadata = metadataList.get(0);
assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE));
assertEquals("my-value", metadata.get("my-key"));
- assertArrayEquals(valueArray, metadata.getValues("my-key-multi"));
+ assertArrayEquals(VALUE_ARRAY, metadata.getValues("my-key-multi"));
assertContains("NullPointerException", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION));
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaMimeTypesTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaMimeTypesTest.java
index 567e681..2f76124 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaMimeTypesTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaMimeTypesTest.java
@@ -26,8 +26,6 @@ import java.io.InputStream;
import java.util.List;
import java.util.Map;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
@@ -37,7 +35,6 @@ import org.junit.Test;
public class TikaMimeTypesTest extends CXFTestBase {
- private static final Gson GSON = new GsonBuilder().create();
private static final String MIMETYPES_PATH = "/mime-types";
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java
similarity index 57%
copy from tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
copy to tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java
index 554aee1..e1812dc 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java
@@ -16,56 +16,61 @@
*/
package org.apache.tika.server.core;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTupleList;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.core.Response;
+import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
+@Ignore("useful for development...need to turn it into a real unit test")
+public class TikaServerAsyncIntegrationTest extends IntegrationTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(TikaServerEmitterIntegrationTest.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TikaServerAsyncIntegrationTest.class);
+ private static FetchEmitTuple.ON_PARSE_EXCEPTION ON_PARSE_EXCEPTION
+ = FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
private static Path TMP_DIR;
private static Path TMP_OUTPUT_DIR;
private static String TIKA_CONFIG_XML;
private static Path TIKA_CONFIG;
- private static Gson GSON = new GsonBuilder().create();
+ private static final int NUM_FILES = 1000;
private static final String EMITTER_NAME = "fse";
private static final String FETCHER_NAME = "fsf";
+ private static List<String> FILE_LIST = new ArrayList<>();
private static String[] FILES = new String[] {
- "hello_world.xml",
- "heavy_hang_30000.xml", "real_oom.xml", "system_exit.xml"
+ "hello_world.xml", "null_pointer.xml"
+ // "heavy_hang_30000.xml", "real_oom.xml", "system_exit.xml"
};
@BeforeClass
@@ -76,10 +81,16 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
Files.createDirectories(inputDir);
Files.createDirectories(TMP_OUTPUT_DIR);
- for (String mockFile : FILES) {
- Files.copy(TikaEmitterTest.class.getResourceAsStream(
- "/test-documents/mock/"+mockFile),
- inputDir.resolve(mockFile));
+ for (int i = 0; i < NUM_FILES; i++) {
+ for (String mockFile : FILES) {
+ String targetName = i+"-"+mockFile;
+ Path target = inputDir.resolve(targetName);
+ FILE_LIST.add(targetName);
+ Files.copy(TikaEmitterTest.class.getResourceAsStream(
+ "/test-documents/mock/" + mockFile),
+ target);
+
+ }
}
TIKA_CONFIG = TMP_DIR.resolve("tika-config.xml");
@@ -134,7 +145,7 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
TikaServerCli.main(
new String[]{
"-enableUnsecureFeatures",
- "-maxFiles", "2000",
+ "-maxFiles", "10000",
"-p", INTEGRATION_TEST_PORT,
"-tmpFilePrefix", "basic-",
"-config", TIKA_CONFIG.toAbsolutePath().toString()
@@ -142,89 +153,29 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
}
};
serverThread.start();
- try {
- testOne("hello_world.xml", true);
- } finally {
- serverThread.interrupt();
- }
- }
- @Test(expected = ProcessingException.class)
- public void testSystemExit() throws Exception {
-
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
try {
- testOne("system_exit.xml", false);
- } finally {
- serverThread.interrupt();
- }
- }
-
- @Test
- public void testOOM() throws Exception {
-
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-JXmx128m",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
+ JsonNode response = sendAsync(FILE_LIST);
+ System.out.println(response);
+ int expected = (ON_PARSE_EXCEPTION == FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT) ?
+ FILE_LIST.size() : FILE_LIST.size()/2;
+ int targets = 0;
+ while (targets < FILE_LIST.size()) {
+ System.out.println("targets "+targets);
+ targets = countTargets();
+ Thread.sleep(1000);
}
- };
- serverThread.start();
- try {
- JsonObject response = testOne("real_oom.xml", false);
- assertContains("heap space", response.get("parse_error").getAsString());
+ System.out.println("TARGETS: "+targets);
} finally {
serverThread.interrupt();
}
}
- @Test(expected = ProcessingException.class)
- public void testTimeout() throws Exception {
-
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-JXmx128m",
- "-taskTimeoutMillis", "2000", "-taskPulseMillis", "100",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
- try {
- JsonObject response = testOne("heavy_hang_30000.xml", false);
- assertContains("heap space", response.get("parse_error").getAsString());
- } finally {
- serverThread.interrupt();
- }
+ private int countTargets() {
+ return TMP_OUTPUT_DIR.toFile().listFiles().length;
}
+
private void awaitServerStartup() throws Exception {
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
@@ -240,7 +191,7 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
}
LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
- } catch (javax.ws.rs.ProcessingException e) {
+ } catch (ProcessingException e) {
LOG.debug("tika test client failed to connect to server", e);
}
@@ -251,25 +202,28 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
elapsed + " ms");
}
- private JsonObject testOne(String fileName, boolean shouldFileExist) throws Exception {
+ private JsonNode sendAsync(List<String> fileNames) throws Exception {
awaitServerStartup();
+ List<FetchEmitTuple> tuples = new ArrayList<>();
+ for (String f : fileNames) {
+ tuples.add(getFetchEmitTuple(f));
+ }
+ String json = JsonFetchEmitTupleList.toJson(tuples);
+
Response response = WebClient
- .create(endPoint + "/emit")
+ .create(endPoint + "/async")
.accept("application/json")
- .post(getJsonString(fileName));
- if (shouldFileExist) {
- Path targFile = TMP_OUTPUT_DIR.resolve(fileName + ".json");
- assertTrue(Files.size(targFile) > 1);
- }
+ .post(json);
+ System.out.println("status: " + response.getStatus());
Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
- return JsonParser.parseReader(reader).getAsJsonObject();
+ return new ObjectMapper().readTree(reader);
}
- private String getJsonString(String fileName) {
- JsonObject root = new JsonObject();
- root.add("fetcher", new JsonPrimitive(FETCHER_NAME));
- root.add("fetchKey", new JsonPrimitive(fileName));
- root.add("emitter", new JsonPrimitive(EMITTER_NAME));
- return GSON.toJson(root);
+ private FetchEmitTuple getFetchEmitTuple(String fileName) throws IOException {
+ return new FetchEmitTuple(
+ new FetchKey(FETCHER_NAME, fileName),
+ new EmitKey(EMITTER_NAME, ""),
+ new Metadata(), ON_PARSE_EXCEPTION
+ );
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
index 554aee1..bea4cee 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
@@ -16,13 +16,16 @@
*/
package org.apache.tika.server.core;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
+import org.apache.tika.pipes.emitter.EmitKey;
+import org.apache.tika.pipes.fetcher.FetchKey;
+import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -32,23 +35,21 @@ import org.slf4j.LoggerFactory;
import javax.ws.rs.ProcessingException;
import javax.ws.rs.core.Response;
+import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
-import java.util.List;
-import java.util.Random;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
@@ -58,14 +59,14 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
private static Path TMP_OUTPUT_DIR;
private static String TIKA_CONFIG_XML;
private static Path TIKA_CONFIG;
- private static Gson GSON = new GsonBuilder().create();
private static final String EMITTER_NAME = "fse";
private static final String FETCHER_NAME = "fsf";
- private static String[] FILES = new String[] {
+ private static String[] FILES = new String[]{
"hello_world.xml",
- "heavy_hang_30000.xml", "real_oom.xml", "system_exit.xml"
+ "heavy_hang_30000.xml", "real_oom.xml", "system_exit.xml",
+ "null_pointer.xml"
};
@BeforeClass
@@ -78,30 +79,30 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
for (String mockFile : FILES) {
Files.copy(TikaEmitterTest.class.getResourceAsStream(
- "/test-documents/mock/"+mockFile),
+ "/test-documents/mock/" + mockFile),
inputDir.resolve(mockFile));
}
TIKA_CONFIG = TMP_DIR.resolve("tika-config.xml");
- TIKA_CONFIG_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+
- "<properties>"+
- "<fetchers>"+
- "<fetcher class=\"org.apache.tika.pipes.fetcher.FileSystemFetcher\">"+
- "<params>"+
- "<param name=\"name\" type=\"string\">"+FETCHER_NAME+"</param>"+
- "<param name=\"basePath\" type=\"string\">"+inputDir.toAbsolutePath()+"</param>"+
- "</params>"+
- "</fetcher>"+
- "</fetchers>"+
- "<emitters>"+
- "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">"+
- "<params>"+
- "<param name=\"name\" type=\"string\">"+EMITTER_NAME+"</param>"+
-
- "<param name=\"basePath\" type=\"string\">"+ TMP_OUTPUT_DIR.toAbsolutePath()+"</param>"+
- "</params>"+
- "</emitter>"+
- "</emitters>"+
+ TIKA_CONFIG_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
+ "<properties>" +
+ "<fetchers>" +
+ "<fetcher class=\"org.apache.tika.pipes.fetcher.FileSystemFetcher\">" +
+ "<params>" +
+ "<param name=\"name\" type=\"string\">" + FETCHER_NAME + "</param>" +
+ "<param name=\"basePath\" type=\"string\">" + inputDir.toAbsolutePath() + "</param>" +
+ "</params>" +
+ "</fetcher>" +
+ "</fetchers>" +
+ "<emitters>" +
+ "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" +
+ "<params>" +
+ "<param name=\"name\" type=\"string\">" + EMITTER_NAME + "</param>" +
+
+ "<param name=\"basePath\" type=\"string\">" + TMP_OUTPUT_DIR.toAbsolutePath() + "</param>" +
+ "</params>" +
+ "</emitter>" +
+ "</emitters>" +
"</properties>";
FileUtils.write(TIKA_CONFIG.toFile(), TIKA_CONFIG_XML, UTF_8);
@@ -124,7 +125,6 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
}
}
-
@Test
public void testBasic() throws Exception {
@@ -143,7 +143,69 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
};
serverThread.start();
try {
- testOne("hello_world.xml", true);
+ JsonNode node = testOne("hello_world.xml", true);
+ assertEquals("ok", node.get("status").asText());
+ } catch (Exception e) {
+ fail("shouldn't have an exception" + e.getMessage());
+ } finally {
+ serverThread.interrupt();
+ }
+ }
+
+ @Test
+ public void testNPEDefault() throws Exception {
+
+ Thread serverThread = new Thread() {
+ @Override
+ public void run() {
+ TikaServerCli.main(
+ new String[]{
+ "-enableUnsecureFeatures",
+ "-maxFiles", "2000",
+ "-p", INTEGRATION_TEST_PORT,
+ "-tmpFilePrefix", "basic-",
+ "-config", TIKA_CONFIG.toAbsolutePath().toString()
+ });
+ }
+ };
+ serverThread.start();
+ try {
+ JsonNode node = testOne("null_pointer.xml", true);
+ assertEquals("ok", node.get("status").asText());
+ assertContains("java.lang.NullPointerException",
+ node.get("parse_exception").asText());
+ } catch (Exception e) {
+ fail("shouldn't have an exception" + e.getMessage());
+ } finally {
+ serverThread.interrupt();
+ }
+ }
+
+ @Test
+ public void testNPESkip() throws Exception {
+
+ Thread serverThread = new Thread() {
+ @Override
+ public void run() {
+ TikaServerCli.main(
+ new String[]{
+ "-enableUnsecureFeatures",
+ "-maxFiles", "2000",
+ "-p", INTEGRATION_TEST_PORT,
+ "-tmpFilePrefix", "basic-",
+ "-config", TIKA_CONFIG.toAbsolutePath().toString()
+ });
+ }
+ };
+ serverThread.start();
+ try {
+ JsonNode node = testOne("null_pointer.xml", false,
+ FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP);
+ assertEquals("ok", node.get("status").asText());
+ assertContains("java.lang.NullPointerException",
+ node.get("parse_exception").asText());
+ } catch (Exception e) {
+ fail("shouldn't have an exception" + e.getMessage());
} finally {
serverThread.interrupt();
}
@@ -192,8 +254,8 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
};
serverThread.start();
try {
- JsonObject response = testOne("real_oom.xml", false);
- assertContains("heap space", response.get("parse_error").getAsString());
+ JsonNode response = testOne("real_oom.xml", false);
+ assertContains("heap space", response.get("parse_error").asText());
} finally {
serverThread.interrupt();
}
@@ -218,8 +280,7 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
};
serverThread.start();
try {
- JsonObject response = testOne("heavy_hang_30000.xml", false);
- assertContains("heap space", response.get("parse_error").getAsString());
+ JsonNode response = testOne("heavy_hang_30000.xml", false);
} finally {
serverThread.interrupt();
}
@@ -228,14 +289,14 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
private void awaitServerStartup() throws Exception {
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
- WebClient client = WebClient.create(endPoint+"/tika").accept("text/plain");
+ WebClient client = WebClient.create(endPoint + "/tika").accept("text/plain");
while (elapsed < MAX_WAIT_MS) {
try {
Response response = client.get();
if (response.getStatus() == 200) {
elapsed = Duration.between(started, Instant.now()).toMillis();
LOG.info("client observes server successfully started after " +
- elapsed+ " ms");
+ elapsed + " ms");
return;
}
LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
@@ -250,26 +311,36 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
throw new TimeoutException("couldn't connect to server after " +
elapsed + " ms");
}
+ private JsonNode testOne(String fileName, boolean shouldFileExist) throws Exception {
+ return testOne(fileName, shouldFileExist, FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
+ }
+
+ private JsonNode testOne(String fileName, boolean shouldFileExist, FetchEmitTuple.ON_PARSE_EXCEPTION onParseException) throws Exception {
- private JsonObject testOne(String fileName, boolean shouldFileExist) throws Exception {
awaitServerStartup();
Response response = WebClient
.create(endPoint + "/emit")
.accept("application/json")
- .post(getJsonString(fileName));
- if (shouldFileExist) {
+ .post(getJsonString(fileName, onParseException));
+ if (response.getStatus() == 200) {
Path targFile = TMP_OUTPUT_DIR.resolve(fileName + ".json");
- assertTrue(Files.size(targFile) > 1);
+ if (shouldFileExist) {
+ assertTrue(Files.size(targFile) > 1);
+ } else {
+ assertFalse(Files.isRegularFile(targFile));
+ }
+ Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
+ return new ObjectMapper().readTree(reader);
}
- Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
- return JsonParser.parseReader(reader).getAsJsonObject();
+ return null;
}
- private String getJsonString(String fileName) {
- JsonObject root = new JsonObject();
- root.add("fetcher", new JsonPrimitive(FETCHER_NAME));
- root.add("fetchKey", new JsonPrimitive(fileName));
- root.add("emitter", new JsonPrimitive(EMITTER_NAME));
- return GSON.toJson(root);
+ private String getJsonString(String fileName, FetchEmitTuple.ON_PARSE_EXCEPTION onParseException) throws IOException {
+ FetchEmitTuple t = new FetchEmitTuple(
+ new FetchKey(FETCHER_NAME, fileName),
+ new EmitKey(EMITTER_NAME, ""),
+ new Metadata(), onParseException
+ );
+ return JsonFetchEmitTuple.toJson(t);
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
index 8d25d83..9e2d7d5 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
@@ -16,26 +16,24 @@
*/
package org.apache.tika.server.core;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import org.apache.cxf.common.logging.LogUtils;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cxf.jaxrs.client.WebClient;
-import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.serialization.JsonMetadataList;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.ws.rs.ProcessingException;
import javax.ws.rs.core.Response;
+import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.net.ConnectException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@@ -55,7 +53,6 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
private static final Logger LOG = LoggerFactory.getLogger(TikaServerIntegrationTest.class);
-
@Test
public void testBasic() throws Exception {
@@ -64,7 +61,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
public void run() {
TikaServerCli.main(
new String[]{
- "-maxFiles", "2000",
+ "-maxFiles", "100",
"-p", INTEGRATION_TEST_PORT,
"-tmpFilePrefix", "basic-"
});
@@ -203,13 +200,13 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
private String getServerId() throws Exception {
Response response = WebClient
- .create(endPoint + STATUS_PATH)
- .accept("application/json")
- .get();
+ .create(endPoint + STATUS_PATH)
+ .accept("application/json")
+ .get();
String jsonString =
CXFTestBase.getStringFromInputStream((InputStream) response.getEntity());
- JsonObject root = JsonParser.parseString(jsonString).getAsJsonObject();
- return root.get("server_id").getAsJsonPrimitive().getAsString();
+ JsonNode root = new ObjectMapper().readTree(jsonString);
+ return root.get("server_id").asText();
}
private int getNumRestarts() throws Exception {
@@ -219,8 +216,8 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
.get();
String jsonString =
CXFTestBase.getStringFromInputStream((InputStream) response.getEntity());
- JsonObject root = JsonParser.parseString(jsonString).getAsJsonObject();
- return root.get("num_restarts").getAsJsonPrimitive().getAsInt();
+ JsonNode root = new ObjectMapper().readTree(jsonString);
+ return root.get("num_restarts").intValue();
}
@Test
@@ -376,10 +373,10 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
awaitServerStartup();
Response response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_STDOUT_STDERR));
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_STDOUT_STDERR));
Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
assertEquals(1, metadataList.size());
@@ -436,7 +433,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
"-p", INTEGRATION_TEST_PORT,
"-taskTimeoutMillis", "10000", "-taskPulseMillis", "500",
"-pingPulseMillis", "100", "-maxRestarts", "0",
- "-JDlog4j.configuration=file:"+ LOG_FILE.toAbsolutePath(),
+ "-JDlog4j.configuration=file:" + LOG_FILE.toAbsolutePath(),
"-tmpFilePrefix", "tika-server-stderrlogging"
});
}
@@ -469,14 +466,14 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
private void awaitServerStartup() throws Exception {
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
- WebClient client = WebClient.create(endPoint+"/tika").accept("text/plain");
+ WebClient client = WebClient.create(endPoint + "/tika").accept("text/plain");
while (elapsed < MAX_WAIT_MS) {
try {
Response response = client.get();
if (response.getStatus() == 200) {
elapsed = Duration.between(started, Instant.now()).toMillis();
LOG.info("client observes server successfully started after " +
- elapsed+ " ms");
+ elapsed + " ms");
return;
}
LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
@@ -513,7 +510,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
awaitServerStartup();
Random r = new Random();
for (int i = 0; i < 100; i++) {
- System.out.println("FILE # "+i);
+ System.out.println("FILE # " + i);
boolean ex = false;
Response response = null;
String file = TEST_HELLO_WORLD;
@@ -525,7 +522,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
} else if (r.nextFloat() < 0.02) {
file = TEST_HEAVY_HANG;
}
- System.out.println("about to process: "+file);
+ System.out.println("about to process: " + file);
response = WebClient
.create(endPoint + META_PATH)
.accept("application/json")
@@ -556,16 +553,29 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
}
private void testBaseline() throws Exception {
- awaitServerStartup();
- Response response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_HELLO_WORLD));
- Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
- List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
- assertEquals(1, metadataList.size());
- assertEquals("Nikolai Lobachevsky", metadataList.get(0).get("author"));
- assertContains("hello world", metadataList.get(0).get("X-TIKA:content"));
+ int maxTries = 3;
+ int tries = 0;
+ while (++tries < maxTries) {
+ awaitServerStartup();
+ Response response = null;
+
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_HELLO_WORLD));
+ } catch (ProcessingException e) {
+ continue;
+ }
+ if (response.getStatus() == 503) {
+ continue;
+ }
+ Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8);
+ List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
+ assertEquals(1, metadataList.size());
+ assertEquals("Nikolai Lobachevsky", metadataList.get(0).get("author"));
+ assertContains("hello world", metadataList.get(0).get("X-TIKA:content"));
+ }
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerStatusTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerStatusTest.java
index 556ac01..d4b6c6d 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerStatusTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerStatusTest.java
@@ -16,13 +16,11 @@
*/
package org.apache.tika.server.core;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
-import org.apache.tika.server.core.CXFTestBase;
-import org.apache.tika.server.core.ServerStatus;
import org.apache.tika.server.core.resource.TikaServerStatus;
import org.apache.tika.server.core.writer.JSONObjWriter;
import org.junit.Test;
@@ -59,15 +57,15 @@ public class TikaServerStatusTest extends CXFTestBase {
Response response = WebClient.create(endPoint + STATUS_PATH).get();
String jsonString =
getStringFromInputStream((InputStream) response.getEntity());
- JsonObject root = JsonParser.parseString(jsonString).getAsJsonObject();
+ JsonNode root = new ObjectMapper().readTree(jsonString);
assertTrue(root.has("server_id"));
assertTrue(root.has("status"));
assertTrue(root.has("millis_since_last_parse_started"));
assertTrue(root.has("files_processed"));
- assertEquals("OPERATING", root.getAsJsonPrimitive("status").getAsString());
- assertEquals(0, root.getAsJsonPrimitive("files_processed").getAsInt());
- long millis = root.getAsJsonPrimitive("millis_since_last_parse_started").getAsInt();
+ assertEquals("OPERATING", root.get("status").asText());
+ assertEquals(0, root.get("files_processed").intValue());
+ long millis = root.get("millis_since_last_parse_started").longValue();
assertTrue(millis > 0 && millis < 360000);
- assertEquals(SERVER_ID, root.getAsJsonPrimitive("server_id").getAsString());
+ assertEquals(SERVER_ID, root.get("server_id").asText());
}
}