You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2025/12/16 13:47:14 UTC
(tika) branch TIKA-4554 created (now b5b6e0b36)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch TIKA-4554
in repository https://gitbox.apache.org/repos/asf/tika.git
at b5b6e0b36 TIKA-4554 - rm ForkParser
This branch includes the following new commits:
new b5b6e0b36 TIKA-4554 - rm ForkParser
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tika) 01/01: TIKA-4554 - rm ForkParser
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4554
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b5b6e0b36a11cb324326e2071890619818d2b209
Author: tallison <ta...@apache.org>
AuthorDate: Tue Dec 16 08:47:02 2025 -0500
TIKA-4554 - rm ForkParser
---
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 30 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 11 -
.../test/java/org/apache/tika/bundle/BundleIT.java | 23 -
.../apache/tika/fork/AutoDetectParserFactory.java | 41 --
.../org/apache/tika/fork/ClassLoaderProxy.java | 153 -----
.../org/apache/tika/fork/ClassLoaderResource.java | 96 ---
.../org/apache/tika/fork/ContentHandlerProxy.java | 199 ------
.../apache/tika/fork/ContentHandlerResource.java | 107 ----
.../main/java/org/apache/tika/fork/ForkClient.java | 355 -----------
.../apache/tika/fork/ForkObjectInputStream.java | 112 ----
.../main/java/org/apache/tika/fork/ForkParser.java | 394 ------------
.../main/java/org/apache/tika/fork/ForkProxy.java | 27 -
.../java/org/apache/tika/fork/ForkResource.java | 27 -
.../main/java/org/apache/tika/fork/ForkServer.java | 302 ----------
.../org/apache/tika/fork/InputStreamProxy.java | 73 ---
.../org/apache/tika/fork/InputStreamResource.java | 52 --
.../org/apache/tika/fork/MemoryURLConnection.java | 42 --
.../apache/tika/fork/MemoryURLStreamHandler.java | 70 ---
.../tika/fork/MemoryURLStreamHandlerFactory.java | 32 -
.../apache/tika/fork/MemoryURLStreamRecord.java | 27 -
.../apache/tika/fork/MetadataContentHandler.java | 42 --
.../java/org/apache/tika/fork/ParserFactory.java | 38 --
.../org/apache/tika/fork/ParserFactoryFactory.java | 55 --
.../main/java/org/apache/tika/fork/ParserShim.java | 34 --
.../java/org/apache/tika/fork/ParserWrapper.java | 45 --
.../main/java/org/apache/tika/fork/RPWShim.java | 44 --
.../fork/RecursiveMetadataContentHandlerProxy.java | 155 -----
.../RecursiveMetadataContentHandlerResource.java | 87 ---
.../java/org/apache/tika/fork/TimeoutLimits.java | 43 --
.../java/org/apache/tika/fork/package-info.java | 22 -
.../java/org/apache/tika/fork/ForkParserTest.java | 665 ---------------------
.../apache/tika/fork/ForkParserTikaBinTest.java | 182 ------
.../java/org/apache/tika/fork/ForkTestParser.java | 88 ---
.../tika/fork/UpperCasingContentHandler.java | 38 --
.../fork/unusedpackage/ClassInUnusedPackage.java | 20 -
.../tika/parser/mock/AutoDetectParserFactory.java | 38 --
.../apache/tika/parser/mock/MockParserFactory.java | 37 --
.../org/apache/tika/parser/pkg/PackageParser.java | 4 +-
.../parser/fork/ForkParserIntegrationTest.java | 360 -----------
39 files changed, 10 insertions(+), 4160 deletions(-)
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index a92972137..411bd070a 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -77,7 +77,6 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.DefaultEmbeddedStreamTranslator;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedStreamTranslator;
-import org.apache.tika.fork.ForkParser;
import org.apache.tika.gui.TikaGUI;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.language.detect.LanguageHandler;
@@ -204,7 +203,6 @@ public class TikaCLI {
private String password = System.getenv("TIKA_PASSWORD");
private Digester digester = null;
private boolean pipeMode = true;
- private boolean fork = false;
private boolean prettyPrint;
private final OutputType XML = new OutputType() {
@Override
@@ -430,8 +428,6 @@ public class TikaCLI {
convertConfigXmlToJson(arg.substring("--convert-config-xml-to-json=".length()));
} else if (arg.equals("--container-aware") || arg.equals("--container-aware-detector")) {
// ignore, as container-aware detectors are now always used
- } else if (arg.equals("-f") || arg.equals("--fork")) {
- fork = true;
} else if (arg.startsWith("--config=")) {
configFilePath = arg.substring("--config=".length());
} else if (arg.startsWith("--digest=")) {
@@ -597,7 +593,6 @@ public class TikaCLI {
out.println(" -V or --version Print the Apache Tika version number");
out.println();
out.println(" -g or --gui Start the Apache Tika GUI");
- out.println(" -f or --fork Use Fork Mode for out-of-process extraction");
out.println();
out.println(" --config=<tika-config.xml>");
out.println(" TikaConfig file. Must be specified before -g, -s, -f or the dump-x-config !");
@@ -1106,24 +1101,15 @@ public class TikaCLI {
private class OutputType {
public void process(TikaInputStream tis, OutputStream output, Metadata metadata) throws Exception {
Parser p = parser;
- if (fork) {
- p = new ForkParser(TikaCLI.class.getClassLoader(), p);
- }
ContentHandler handler = getContentHandler(output, metadata);
- try {
- p.parse(tis, handler, metadata, context);
- // fix for TIKA-596: if a parser doesn't generate
- // XHTML output, the lack of an output document prevents
- // metadata from being output: this fixes that
- if (handler instanceof NoDocumentMetHandler) {
- NoDocumentMetHandler metHandler = (NoDocumentMetHandler) handler;
- if (!metHandler.metOutput()) {
- metHandler.endDocument();
- }
- }
- } finally {
- if (fork) {
- ((ForkParser) p).close();
+ p.parse(tis, handler, metadata, context);
+ // fix for TIKA-596: if a parser doesn't generate
+ // XHTML output, the lack of an output document prevents
+ // metadata from being output: this fixes that
+ if (handler instanceof NoDocumentMetHandler) {
+ NoDocumentMetHandler metHandler = (NoDocumentMetHandler) handler;
+ if (!metHandler.metOutput()) {
+ metHandler.endDocument();
}
}
}
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
index 4f658a122..73e167d60 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
@@ -200,17 +200,6 @@ public class TikaCLITest {
assertTrue(out.contains("body"));
}
- /**
- * Tests -f option of the cli
- *
- * @throws Exception
- */
- @Test
- public void testForkParser() throws Exception {
- String content = getParamOutContent("-f", resourcePrefix + "alice.cli.test");
- assertTrue(content.contains("finished off the cake"));
- }
-
/**
* Tests -m option of the cli
*
diff --git a/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java b/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
index b82e040d9..58659d4f8 100644
--- a/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
+++ b/tika-bundles/tika-bundle-standard/src/test/java/org/apache/tika/bundle/BundleIT.java
@@ -60,7 +60,6 @@ import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.fork.ForkParser;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -153,28 +152,6 @@ public class BundleIT {
assertEquals(MediaType.application("pdf"), contentTypeDetector.detect(null, metadataPDF, new ParseContext()));
}
- @Test
- public void testForkParser() throws Exception {
- try (ForkParser parser = new ForkParser(Activator.class.getClassLoader(), defaultParser)) {
- String data =
- "<!DOCTYPE html>\n<html><body><p>test <span>content</span></p></body></html>";
- try (TikaInputStream tis = TikaInputStream.get(data.getBytes(UTF_8))) {
- Writer writer = new StringWriter();
- ContentHandler contentHandler = new BodyContentHandler(writer);
- Metadata metadata = new Metadata();
- MediaType type = contentTypeDetector.detect(tis, metadata, new ParseContext());
- assertEquals(type.toString(), "text/html");
- metadata.add(Metadata.CONTENT_TYPE, type.toString());
- ParseContext parseCtx = new ParseContext();
- parser.parse(tis, contentHandler, metadata, parseCtx);
- writer.flush();
- String content = writer.toString();
- assertTrue(content.length() > 0);
- assertEquals("test content", content.trim());
- }
- }
- }
-
@Test
public void testBundleSimpleText() throws Exception {
Tika tika = new Tika();
diff --git a/tika-core/src/main/java/org/apache/tika/fork/AutoDetectParserFactory.java b/tika-core/src/main/java/org/apache/tika/fork/AutoDetectParserFactory.java
deleted file mode 100644
index ed8bd4627..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/AutoDetectParserFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.Parser;
-
-/**
- * Factory for an AutoDetectParser
- */
-public class AutoDetectParserFactory extends ParserFactory {
-
- public AutoDetectParserFactory(Map<String, String> args) {
- super(args);
- }
-
- @Override
- public Parser build() throws IOException, SAXException, TikaException {
- return new AutoDetectParser();
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ClassLoaderProxy.java b/tika-core/src/main/java/org/apache/tika/fork/ClassLoaderProxy.java
deleted file mode 100644
index 51b1beeff..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ClassLoaderProxy.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-class ClassLoaderProxy extends ClassLoader implements ForkProxy {
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = -7303109260448540420L;
-
- /**
- * Names of resources that could not be found. Used to avoid repeated
- * lookup of commonly accessed, but often not present, resources like
- * <code>META-INF/services/javax.xml.parsers.SAXParserFactory</code>.
- */
- private final Set<String> notFound = new HashSet<>();
-
- private final int resource;
-
- private transient DataInputStream input;
-
- private transient DataOutputStream output;
-
- public ClassLoaderProxy(int resource) {
- this.resource = resource;
- }
-
- public void init(DataInputStream input, DataOutputStream output) {
- this.input = input;
- this.output = output;
- }
-
- @Override
- protected synchronized URL findResource(String name) {
- if (notFound.contains(name)) {
- return null;
- }
- try {
- // Send a request to load the resource data
- output.write(ForkServer.RESOURCE);
- output.write(resource);
- output.write(1);
- output.writeUTF(name);
- output.flush();
-
- // Receive the response
- if (input.readBoolean()) {
- return MemoryURLStreamHandler.createURL(readStream());
- } else {
- notFound.add(name);
- return null;
- }
- } catch (IOException e) {
- return null;
- }
- }
-
- @Override
- protected synchronized Enumeration<URL> findResources(String name) throws IOException {
- // Send a request to load the resources
- output.write(ForkServer.RESOURCE);
- output.write(resource);
- output.write(2);
- output.writeUTF(name);
- output.flush();
-
- // Receive the response
- List<URL> resources = new ArrayList<>();
- while (input.readBoolean()) {
- resources.add(MemoryURLStreamHandler.createURL(readStream()));
- }
- return Collections.enumeration(resources);
- }
-
- @Override
- protected synchronized Class<?> findClass(String name) throws ClassNotFoundException {
- try {
- // Send a request to load the class data
- output.write(ForkServer.RESOURCE);
- output.write(resource);
- output.write(1);
- output.writeUTF(name.replace('.', '/') + ".class");
- output.flush();
-
- // Receive the response
- if (input.readBoolean()) {
- byte[] data = readStream();
- Class<?> clazz = defineClass(name, data, 0, data.length);
- definePackageIfNecessary(name, clazz);
- return clazz;
- } else {
- throw new ClassNotFoundException("Unable to find class " + name);
- }
- } catch (IOException e) {
- throw new ClassNotFoundException("Unable to load class " + name, e);
- }
- }
-
- private void definePackageIfNecessary(String className, Class<?> clazz) {
- String packageName = toPackageName(className);
- if (packageName != null && getDefinedPackage(packageName) == null) {
- definePackage(packageName, null, null, null, null, null, null, null);
- }
- }
-
- private String toPackageName(String className) {
- int packageEndIndex = className.lastIndexOf('.');
- if (packageEndIndex > 0) {
- return className.substring(0, packageEndIndex);
- }
- return null;
- }
-
- private byte[] readStream() throws IOException {
- try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
- byte[] buffer = new byte[0xffff];
- int n;
- while ((n = input.readUnsignedShort()) > 0) {
- input.readFully(buffer, 0, n);
- stream.write(buffer, 0, n);
- }
- return stream.toByteArray();
- }
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ClassLoaderResource.java b/tika-core/src/main/java/org/apache/tika/fork/ClassLoaderResource.java
deleted file mode 100644
index 7af85ada5..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ClassLoaderResource.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.Enumeration;
-
-class ClassLoaderResource implements ForkResource {
-
- private final ClassLoader loader;
-
- public ClassLoaderResource(ClassLoader loader) {
- this.loader = loader;
- }
-
- /**
- * Processes a request for one (code 1) or many (code 2) class loader
- * resources. The requested resources are sent preceded with a boolean
- * <code>true</code> value. If the resource was not found (code 1) or
- * when the last resource has been sent (code 2), a boolean
- * <code>false</code> value is sent instead.
- *
- * @param name resource name
- * @throws IOException if the resource could not be sent
- */
- public Throwable process(DataInputStream input, DataOutputStream output) throws IOException {
- byte type = input.readByte();
- String name = input.readUTF();
- if (type == 1) {
- InputStream stream = loader.getResourceAsStream(name);
- if (stream != null) {
- output.writeBoolean(true);
- writeAndCloseStream(output, stream);
- } else {
- output.writeBoolean(false);
- }
- } else if (type == 2) {
- Enumeration<URL> resources = loader.getResources(name);
- while (resources.hasMoreElements()) {
- output.writeBoolean(true);
- InputStream stream = resources.nextElement().openStream();
- writeAndCloseStream(output, stream);
- }
- output.writeBoolean(false);
- }
- output.flush();
- return null;
- }
-
- /**
- * Sends the contents of the given input stream to the given output.
- * The stream is sent in chunks of less than 64kB, each preceded by
- * a 16-bit integer value that indicates the length of the following
- * chunk. A zero short value is sent at the end to signify the end of
- * the stream.
- * <p>
- * The stream is guaranteed to be closed by this method, regardless of
- * the way it returns.
- *
- * @param stream the stream to be sent
- * @throws IOException if the stream could not be sent
- */
- private void writeAndCloseStream(DataOutputStream output, InputStream stream)
- throws IOException {
- try {
- byte[] buffer = new byte[0x10000 - 1];
- int n;
- while ((n = stream.read(buffer)) != -1) {
- output.writeShort(n);
- output.write(buffer, 0, n);
- }
- output.writeShort(0);
- } finally {
- stream.close();
- }
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ContentHandlerProxy.java b/tika-core/src/main/java/org/apache/tika/fork/ContentHandlerProxy.java
deleted file mode 100644
index 371dd05c5..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ContentHandlerProxy.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.Locator;
-import org.xml.sax.SAXException;
-
-class ContentHandlerProxy implements ContentHandler, ForkProxy {
-
- public static final int START_DOCUMENT = 1;
- public static final int END_DOCUMENT = 2;
- public static final int START_PREFIX_MAPPING = 3;
- public static final int END_PREFIX_MAPPING = 4;
- public static final int START_ELEMENT = 5;
- public static final int END_ELEMENT = 6;
- public static final int CHARACTERS = 7;
- public static final int IGNORABLE_WHITESPACE = 8;
- public static final int PROCESSING_INSTRUCTION = 9;
- public static final int SKIPPED_ENTITY = 10;
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = 737511106054617524L;
-
- private final int resource;
-
- private transient DataOutputStream output;
-
- public ContentHandlerProxy(int resource) {
- this.resource = resource;
- }
-
- public void init(DataInputStream input, DataOutputStream output) {
- this.output = output;
- }
-
- private void sendRequest(int type) throws SAXException {
- try {
- output.writeByte(ForkServer.RESOURCE);
- output.writeByte(resource);
- output.writeByte(type);
- } catch (IOException e) {
- throw new SAXException("Unexpected fork proxy problem", e);
- }
- }
-
- private void sendString(String string) throws SAXException {
- try {
- if (string != null) {
- output.writeBoolean(true);
- writeString(string);
- } else {
- output.writeBoolean(false);
- }
- } catch (IOException e) {
- throw new SAXException("Unexpected fork proxy problem", e);
- }
- }
-
- /**
- * Breaks the string in 21,845 size chunks to not
- * throw UTFDataFormatException at least in Oracle JDK 8.
- */
- private void writeString(String string) throws IOException {
- int max = 65535 / 3;
- int frags = (int) Math.ceil((double) string.length() / max);
- output.writeInt(frags);
- int i = 0;
- while (i < frags) {
- int end = (i < frags - 1) ? (i + 1) * max : string.length();
- output.writeUTF(string.substring(i * max, end));
- i++;
- }
- }
-
- private void sendCharacters(char[] ch, int start, int length) throws SAXException {
- try {
- writeString(new String(ch, start, length));
-
- } catch (IOException e) {
- throw new SAXException("Unexpected fork proxy problem", e);
- }
- }
-
- private void doneSending() throws SAXException {
- try {
- output.flush();
- } catch (IOException e) {
- throw new SAXException("Unexpected fork proxy problem", e);
- }
- }
-
- public void setDocumentLocator(Locator locator) {
- // skip
- }
-
- public void startDocument() throws SAXException {
- sendRequest(START_DOCUMENT);
- doneSending();
- }
-
- public void endDocument() throws SAXException {
- sendRequest(END_DOCUMENT);
- doneSending();
- }
-
- public void startPrefixMapping(String prefix, String uri) throws SAXException {
- sendRequest(START_PREFIX_MAPPING);
- sendString(prefix);
- sendString(uri);
- doneSending();
- }
-
- public void endPrefixMapping(String prefix) throws SAXException {
- sendRequest(END_PREFIX_MAPPING);
- sendString(prefix);
- doneSending();
- }
-
- public void startElement(String uri, String localName, String qName, Attributes atts)
- throws SAXException {
- sendRequest(START_ELEMENT);
- sendString(uri);
- sendString(localName);
- sendString(qName);
- int n = -1;
- if (atts != null) {
- n = atts.getLength();
- }
- try {
- output.writeInt(n);
- } catch (IOException e) {
- throw new SAXException("Unexpected fork proxy problem", e);
- }
- for (int i = 0; i < n; i++) {
- sendString(atts.getURI(i));
- sendString(atts.getLocalName(i));
- sendString(atts.getQName(i));
- sendString(atts.getType(i));
- sendString(atts.getValue(i));
- }
- doneSending();
- }
-
- public void endElement(String uri, String localName, String qName) throws SAXException {
- sendRequest(END_ELEMENT);
- sendString(uri);
- sendString(localName);
- sendString(qName);
- doneSending();
- }
-
- public void characters(char[] ch, int start, int length) throws SAXException {
- sendRequest(CHARACTERS);
- sendCharacters(ch, start, length);
- doneSending();
- }
-
- public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
- sendRequest(IGNORABLE_WHITESPACE);
- sendCharacters(ch, start, length);
- doneSending();
- }
-
- public void processingInstruction(String target, String data) throws SAXException {
- sendRequest(PROCESSING_INSTRUCTION);
- sendString(target);
- sendString(data);
- doneSending();
- }
-
- public void skippedEntity(String name) throws SAXException {
- sendRequest(SKIPPED_ENTITY);
- sendString(name);
- doneSending();
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ContentHandlerResource.java b/tika-core/src/main/java/org/apache/tika/fork/ContentHandlerResource.java
deleted file mode 100644
index f8971b9a6..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ContentHandlerResource.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.AttributesImpl;
-
-class ContentHandlerResource implements ForkResource {
-
- private final ContentHandler handler;
-
- public ContentHandlerResource(ContentHandler handler) {
- this.handler = handler;
- }
-
- public Throwable process(DataInputStream input, DataOutputStream output) throws IOException {
- try {
- internalProcess(input);
- return null;
- } catch (SAXException e) {
- return e;
- }
- }
-
- private void internalProcess(DataInputStream input) throws IOException, SAXException {
- int type = input.readUnsignedByte();
- if (type == ContentHandlerProxy.START_DOCUMENT) {
- handler.startDocument();
- } else if (type == ContentHandlerProxy.END_DOCUMENT) {
- handler.endDocument();
- } else if (type == ContentHandlerProxy.START_PREFIX_MAPPING) {
- handler.startPrefixMapping(readString(input), readString(input));
- } else if (type == ContentHandlerProxy.END_PREFIX_MAPPING) {
- handler.endPrefixMapping(readString(input));
- } else if (type == ContentHandlerProxy.START_ELEMENT) {
- String uri = readString(input);
- String localName = readString(input);
- String qName = readString(input);
- AttributesImpl atts = null;
- int n = input.readInt();
- if (n >= 0) {
- atts = new AttributesImpl();
- for (int i = 0; i < n; i++) {
- atts.addAttribute(readString(input), readString(input), readString(input),
- readString(input), readString(input));
- }
- }
- handler.startElement(uri, localName, qName, atts);
- } else if (type == ContentHandlerProxy.END_ELEMENT) {
- String uri = readString(input);
- String localName = readString(input);
- String qName = readString(input);
- handler.endElement(uri, localName, qName);
- } else if (type == ContentHandlerProxy.CHARACTERS) {
- char[] ch = readCharacters(input);
- handler.characters(ch, 0, ch.length);
- } else if (type == ContentHandlerProxy.IGNORABLE_WHITESPACE) {
- char[] ch = readCharacters(input);
- handler.characters(ch, 0, ch.length);
- } else if (type == ContentHandlerProxy.PROCESSING_INSTRUCTION) {
- handler.processingInstruction(readString(input), readString(input));
- } else if (type == ContentHandlerProxy.SKIPPED_ENTITY) {
- handler.skippedEntity(readString(input));
- }
- }
-
- private String readString(DataInputStream input) throws IOException {
- if (input.readBoolean()) {
- return readStringUTF(input);
- } else {
- return null;
- }
- }
-
- private char[] readCharacters(DataInputStream input) throws IOException {
- return readStringUTF(input).toCharArray();
- }
-
- private String readStringUTF(DataInputStream input) throws IOException {
- int frags = input.readInt();
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < frags; i++) {
- sb.append(input.readUTF());
- }
- return sb.toString();
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ForkClient.java b/tika-core/src/main/java/org/apache/tika/fork/ForkClient.java
deleted file mode 100644
index f1a47206d..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ForkClient.java
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.NotSerializableException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-import java.util.zip.ZipEntry;
-
-import org.apache.commons.io.IOUtils;
-import org.xml.sax.ContentHandler;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
-import org.apache.tika.sax.RecursiveParserWrapperHandler;
-import org.apache.tika.utils.ProcessUtils;
-
-class ForkClient {
- private static final AtomicInteger CLIENT_COUNTER = new AtomicInteger(0);
-
- private final List<ForkResource> resources = new ArrayList<>();
-
- private final ClassLoader loader;
-
- private final File jar;
-
- private final Process process;
-
- private final DataOutputStream output;
-
- private final DataInputStream input;
-
- //this is used for debugging/smoke testing
- private final int id = CLIENT_COUNTER.incrementAndGet();
-
- private volatile int filesProcessed = 0;
-
- public ForkClient(Path tikaDir, ParserFactoryFactory parserFactoryFactory, List<String> java,
- TimeoutLimits timeoutLimits) throws IOException, TikaException {
- this(tikaDir, parserFactoryFactory, null, java, timeoutLimits);
- }
-
- /**
- * @param tikaDir directory containing jars from which to start
- * the child server and load the Parser
- * @param parserFactoryFactory factory to send to forked process to build parser
- * upon arrival
- * @param classLoader class loader to use for non-parser resource
- * (content-handler, etc.)
- * @param java java commandline to use for the commandline server
- * @throws IOException
- * @throws TikaException
- */
- public ForkClient(Path tikaDir, ParserFactoryFactory parserFactoryFactory,
- ClassLoader classLoader, List<String> java, TimeoutLimits timeoutLimits)
- throws IOException, TikaException {
- jar = null;
- loader = null;
- boolean ok = false;
- ProcessBuilder builder = new ProcessBuilder();
- List<String> command = new ArrayList<>(java);
- command.add("-cp");
- String dirString = tikaDir.toAbsolutePath().toString();
- if (!dirString.endsWith("/")) {
- dirString += "/*";
- } else {
- dirString += "/";
- }
- dirString = ProcessUtils.escapeCommandLine(dirString);
- command.add(dirString);
- command.add("org.apache.tika.fork.ForkServer");
- command.add(Long.toString(timeoutLimits.getPulseMS()));
- command.add(Long.toString(timeoutLimits.getParseTimeoutMS()));
- command.add(Long.toString(timeoutLimits.getWaitTimeoutMS()));
- builder.command(command);
- builder.redirectError(ProcessBuilder.Redirect.INHERIT);
- try {
- this.process = builder.start();
-
- this.output = new DataOutputStream(process.getOutputStream());
- this.input = new DataInputStream(process.getInputStream());
-
- waitForStartBeacon();
- if (classLoader != null) {
- output.writeByte(ForkServer.INIT_PARSER_FACTORY_FACTORY_LOADER);
- } else {
- output.writeByte(ForkServer.INIT_PARSER_FACTORY_FACTORY);
- }
- output.flush();
- sendObject(parserFactoryFactory, resources);
- if (classLoader != null) {
- sendObject(classLoader, resources);
- }
- waitForStartBeacon();
- ok = true;
- } catch (Throwable t) {
- t.printStackTrace();
- throw t;
- } finally {
- if (!ok) {
- close();
- }
- }
- }
-
-
- public ForkClient(ClassLoader loader, Object object, List<String> java,
- TimeoutLimits timeoutLimits) throws IOException, TikaException {
- boolean ok = false;
- try {
- this.loader = loader;
- this.jar = createBootstrapJar();
-
- ProcessBuilder builder = new ProcessBuilder();
- List<String> command = new ArrayList<>(java);
- command.add("-jar");
- command.add(jar.getPath());
- command.add(Long.toString(timeoutLimits.getPulseMS()));
- command.add(Long.toString(timeoutLimits.getParseTimeoutMS()));
- command.add(Long.toString(timeoutLimits.getWaitTimeoutMS()));
- builder.command(command);
- builder.redirectError(ProcessBuilder.Redirect.INHERIT);
- this.process = builder.start();
-
- this.output = new DataOutputStream(process.getOutputStream());
- this.input = new DataInputStream(process.getInputStream());
-
- waitForStartBeacon();
- output.writeByte(ForkServer.INIT_LOADER_PARSER);
- output.flush();
- sendObject(loader, resources);
- sendObject(object, resources);
- waitForStartBeacon();
-
- ok = true;
- } finally {
- if (!ok) {
- close();
- }
- }
- }
-
- /**
- * Creates a temporary jar file that can be used to bootstrap the forked
- * server process. Remember to remove the file when no longer used.
- *
- * @return the created jar file
- * @throws IOException if the bootstrap archive could not be created
- */
- private static File createBootstrapJar() throws IOException {
- File file = Files.createTempFile("apache-tika-fork-", ".jar").toFile();
- boolean ok = false;
- try {
- fillBootstrapJar(file);
- ok = true;
- } finally {
- if (!ok) {
- file.delete();
- }
- }
- return file;
- }
-
- /**
- * Fills in the jar file used to bootstrap the forked server process.
- * All the required <code>.class</code> files and a manifest with a
- * <code>Main-Class</code> entry are written into the archive.
- *
- * @param file file to hold the bootstrap archive
- * @throws IOException if the bootstrap archive could not be created
- */
- private static void fillBootstrapJar(File file) throws IOException {
- try (JarOutputStream jar = new JarOutputStream(new FileOutputStream(file))) {
- String manifest = "Main-Class: " + ForkServer.class.getName() + "\n";
- jar.putNextEntry(new ZipEntry("META-INF/MANIFEST.MF"));
- jar.write(manifest.getBytes(UTF_8));
-
- Class<?>[] bootstrap = {ForkServer.class, ForkObjectInputStream.class, ForkProxy.class,
- ClassLoaderProxy.class, MemoryURLConnection.class, MemoryURLStreamHandler.class,
- MemoryURLStreamHandlerFactory.class, MemoryURLStreamRecord.class,
- TikaException.class};
- ClassLoader loader = ForkServer.class.getClassLoader();
- for (Class<?> klass : bootstrap) {
- String path = klass.getName().replace('.', '/') + ".class";
- try (InputStream input = loader.getResourceAsStream(path)) {
- jar.putNextEntry(new JarEntry(path));
- IOUtils.copy(input, jar);
- }
- }
- }
- }
-
- private void waitForStartBeacon() throws IOException {
- while (true) {
- int type = input.read();
- if ((byte) type == ForkServer.READY) {
- return;
- } else if ((byte) type == ForkServer.FAILED_TO_START) {
- throw new IOException("Server had a catastrophic initialization failure");
- } else if (type == -1) {
- throw new IOException("EOF while waiting for start beacon");
- } else {
- //can't do this because of
- // ForkParserIntegrationTest
- // #testAttachingADebuggerOnTheForkedParserShouldWork
-// throw new IOException("Unexpected byte while waiting for start beacon: "+type);
- }
- }
- }
-
- public synchronized boolean ping() {
- try {
- output.writeByte(ForkServer.PING);
- output.flush();
- while (true) {
- int type = input.read();
- return type == ForkServer.PING;
- }
- } catch (IOException e) {
- return false;
- }
- }
-
- public synchronized Throwable call(String method, Object... args)
- throws IOException, TikaException {
- filesProcessed++;
- List<ForkResource> r = new ArrayList<>(resources);
- output.writeByte(ForkServer.CALL);
- output.writeUTF(method);
- for (Object arg : args) {
- sendObject(arg, r);
- }
- return waitForResponse(r);
- }
-
- public int getFilesProcessed() {
- return filesProcessed;
- }
-
- /**
- * Serializes the object first into an in-memory buffer and then
- * writes it to the output stream with a preceding size integer.
- *
- * @param object object to be serialized
- * @param resources list of fork resources, used when adding proxies
- * @throws IOException if the object could not be serialized
- */
- private void sendObject(Object object, List<ForkResource> resources)
- throws IOException, TikaException {
- int n = resources.size();
- if (object instanceof InputStream) {
- resources.add(new InputStreamResource((InputStream) object));
- object = new InputStreamProxy(n);
- } else if (object instanceof RecursiveParserWrapperHandler) {
- resources.add(new RecursiveMetadataContentHandlerResource(
- (RecursiveParserWrapperHandler) object));
- object = new RecursiveMetadataContentHandlerProxy(n,
- ((RecursiveParserWrapperHandler) object).getContentHandlerFactory());
- } else if (object instanceof ContentHandler &&
- !(object instanceof AbstractRecursiveParserWrapperHandler)) {
- resources.add(new ContentHandlerResource((ContentHandler) object));
- object = new ContentHandlerProxy(n);
- } else if (object instanceof ClassLoader) {
- resources.add(new ClassLoaderResource((ClassLoader) object));
- object = new ClassLoaderProxy(n);
- }
-
- try {
- ForkObjectInputStream.sendObject(object, output);
- } catch (NotSerializableException nse) {
- // Build a more friendly error message for this
- throw new TikaException("Unable to serialize " + object.getClass().getSimpleName() +
- " to pass to the Forked Parser", nse);
- }
-
- waitForResponse(resources);
- }
-
- public synchronized void close() {
- try {
- if (output != null) {
- output.close();
- }
- if (input != null) {
- input.close();
- }
- } catch (IOException ignore) {
- }
- if (process != null) {
- process.destroyForcibly();
- try {
- //TIKA-1933
- process.waitFor();
- } catch (InterruptedException e) {
- //swallow
- }
- }
- if (jar != null) {
- jar.delete();
- }
- }
-
- private Throwable waitForResponse(List<ForkResource> resources) throws IOException {
- output.flush();
- while (!Thread.currentThread().isInterrupted()) {
- int type = input.read();
- if (type == -1) {
- throw new IOException("Lost connection to a forked server process");
- } else if (type == ForkServer.RESOURCE) {
- ForkResource resource = resources.get(input.readUnsignedByte());
- resource.process(input, output);
- } else if ((byte) type == ForkServer.ERROR) {
- try {
- return (Throwable) ForkObjectInputStream.readObject(input, loader);
- } catch (ClassNotFoundException e) {
- throw new IOException("Unable to deserialize an exception", e);
- }
- } else {
- return null;
- }
- }
- throw new IOException(new InterruptedException());
- }
-
- public int getId() {
- return id;
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ForkObjectInputStream.java b/tika-core/src/main/java/org/apache/tika/fork/ForkObjectInputStream.java
deleted file mode 100644
index 61e2dae48..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ForkObjectInputStream.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.ObjectStreamClass;
-
-/**
- * An object input stream that uses a given class loader when deserializing
- * objects.
- * <p>
- * Note that this functionality could easily be implemented as a simple
- * anonymous {@link ObjectInputStream} subclass, but since the
- * functionality is needed during the somewhat complicated bootstrapping
- * of the stdin/out communication channel of a forked server process,
- * it's better if class has a stable name that can be referenced at
- * compile-time by the {@link ForkClient} class.
- */
-class ForkObjectInputStream extends ObjectInputStream {
-
- /**
- * The class loader used when deserializing objects.
- */
- private final ClassLoader loader;
-
- /**
- * Creates a new object input stream that uses the given class loader
- * when deserializing objects.
- *
- * @param input underlying input stream
- * @param loader class loader used when deserializing objects
- * @throws IOException if this stream could not be initiated
- */
- public ForkObjectInputStream(InputStream input, ClassLoader loader) throws IOException {
- super(input);
- this.loader = loader;
- }
-
- /**
- * Serializes the object first into an in-memory buffer and then
- * writes it to the output stream with a preceding size integer.
- *
- * @param object object to be serialized
- * @param output output stream
- * @throws IOException if the object could not be serialized
- */
- public static void sendObject(Object object, DataOutputStream output) throws IOException {
- ByteArrayOutputStream buffer = new ByteArrayOutputStream();
- try (ObjectOutputStream serializer = new ObjectOutputStream(buffer)) {
- serializer.writeObject(object);
- }
-
- byte[] data = buffer.toByteArray();
- output.writeInt(data.length);
- output.write(data);
- }
-
- /**
- * Deserializes an object from the given stream. The serialized object
- * is expected to be preceded by a size integer, that is used for reading
- * the entire serialization into a memory before deserializing it.
- *
- * @param input input stream from which the serialized object is read
- * @param loader class loader to be used for loading referenced classes
- * @throws IOException if the object could not be deserialized
- * @throws ClassNotFoundException if a referenced class is not found
- */
- public static Object readObject(DataInputStream input, ClassLoader loader)
- throws IOException, ClassNotFoundException {
- int n = input.readInt();
- byte[] data = new byte[n];
- input.readFully(data);
-
- ObjectInputStream deserializer =
- new ForkObjectInputStream(new ByteArrayInputStream(data), loader);
- return deserializer.readObject();
- }
-
- /**
- * Loads the identified class from the specified class loader.
- *
- * @param desc class description
- * @return class loaded class
- * @throws ClassNotFoundException if the class can not be found
- */
- @Override
- protected Class<?> resolveClass(ObjectStreamClass desc) throws ClassNotFoundException {
- return Class.forName(desc.getName(), false, loader);
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ForkParser.java b/tika-core/src/main/java/org/apache/tika/fork/ForkParser.java
deleted file mode 100644
index 7020875ac..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ForkParser.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import java.util.Set;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
-import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
-import org.apache.tika.sax.TeeContentHandler;
-
-public class ForkParser implements Parser, Closeable {
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = -4962742892274663950L;
-
- //these are used by the legacy usage
- private final ClassLoader loader;
- private final Parser parser;
-
- //these are used when the server builds a parser via a directory
- //of jars, not via legacy bootstrap etc.
- private final Path tikaBin;
- private final ParserFactoryFactory parserFactoryFactory;
- private final Queue<ForkClient> pool = new LinkedList<>();
- /**
- * Java command line
- */
- private List<String> java = Arrays.asList("java", "-Xmx32m", "-Djava.awt.headless=true");
- /**
- * Process pool size
- */
- private int poolSize = 5;
- private int currentlyInUse = 0;
- private long serverPulseMillis = 1000;
-
- private long serverParseTimeoutMillis = 60000;
-
- private long serverWaitTimeoutMillis = 60000;
-
- private int maxFilesProcessedPerClient = -1;
-
- /**
- * If you have a directory with, say, tike-app.jar and you want the
- * forked process/server to build a parser
- * and run it from that -- so that you can keep all of those dependencies out of
- * your client code, use this initializer.
- *
- * @param tikaBin directory containing the tika-app.jar or similar --
- * full jar including tika-core and all
- * desired parsers and dependencies
- * @param factoryFactory
- */
- public ForkParser(Path tikaBin, ParserFactoryFactory factoryFactory) {
- loader = null;
- parser = null;
- this.tikaBin = tikaBin;
- this.parserFactoryFactory = factoryFactory;
- }
-
- /**
- * <b>EXPERT</b>
- *
- * @param tikaBin directory containing the tika-app.jar or similar
- * -- full jar including tika-core and all
- * desired parsers and dependencies
- * @param parserFactoryFactory -- the factory to use to generate the parser factory
- * in the forked process/server
- * @param classLoader to use for all classes besides the parser in the
- * forked process/server
- */
- public ForkParser(Path tikaBin, ParserFactoryFactory parserFactoryFactory,
- ClassLoader classLoader) {
- parser = null;
- loader = classLoader;
- this.tikaBin = tikaBin;
- this.parserFactoryFactory = parserFactoryFactory;
- }
-
- /**
- * @param loader The ClassLoader to use
- * @param parser the parser to delegate to. This one cannot be another ForkParser
- */
- public ForkParser(ClassLoader loader, Parser parser) {
- if (parser instanceof ForkParser) {
- throw new IllegalArgumentException(
- "The underlying parser of a ForkParser should not be a ForkParser, " +
- "but a specific implementation.");
- }
- if (parser instanceof RecursiveParserWrapper) {
- parser = new RPWShim(((RecursiveParserWrapper) parser).getWrappedParser());
- } else {
- parser = new ParserWrapper(parser);
- }
- this.tikaBin = null;
- this.parserFactoryFactory = null;
- this.loader = loader;
- this.parser = parser;
- }
-
- public ForkParser(ClassLoader loader) {
- this(loader, new AutoDetectParser());
- }
-
- public ForkParser() {
- this(ForkParser.class.getClassLoader());
- }
-
- /**
- * Returns the size of the process pool.
- *
- * @return process pool size
- */
- public synchronized int getPoolSize() {
- return poolSize;
- }
-
- /**
- * Sets the size of the process pool.
- *
- * @param poolSize process pool size
- */
- public synchronized void setPoolSize(int poolSize) {
- this.poolSize = poolSize;
- }
-
- /**
- * Sets the command used to start the forked server process.
- * The arguments "-jar" and "/path/to/bootstrap.jar"
- * or "-cp" and "/path/to/tika_bin" are
- * appended to the given command when starting the process.
- * The default setting is {"java", "-Xmx32m"}.
- * <p/>
- * Creates a defensive copy.
- *
- * @param java java command line
- */
- public void setJavaCommand(List<String> java) {
- this.java = new ArrayList<>(java);
- }
-
- /**
- * Returns the command used to start the forked server process.
- * <p/>
- * Returned list is unmodifiable.
- *
- * @return java command line args
- */
- public List<String> getJavaCommandAsList() {
- return Collections.unmodifiableList(java);
- }
-
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- return parser.getSupportedTypes(context);
- }
-
- /**
- * This sends the objects to the server for parsing, and the server via
- * the proxies acts on the handler as if it were updating it directly.
- * <p>
- * If using a {@link org.apache.tika.parser.RecursiveParserWrapper}, there are two options:
- * </p>
- * <p>
- * <ol>
- * <li>Send in a class that extends
- * {@link org.apache.tika.sax.RecursiveParserWrapperHandler},
- * and the server will proxy back the data as best it can[0].</li>
- * <li>Send in a class that extends {@link AbstractRecursiveParserWrapperHandler}
- * and the server will act on the class but not proxy back the data. This
- * can be used, for example, if all you want to do is write to disc, extend
- * {@link AbstractRecursiveParserWrapperHandler} to write to disc when
- * {@link AbstractRecursiveParserWrapperHandler#endDocument(ContentHandler,
- * Metadata)}
- * is called, and the server will take care of the writing via the handler.</li>
- * </ol>
- * </p>
- * <p>
- * <b>NOTE:</b>[0] "the server will proxy back the data as best it can".
- * If the handler implements Serializable and is actually serializable, the
- * server will send it and the
- * {@link Metadata} back upon
- * {@link org.apache.tika.sax.RecursiveParserWrapperHandler#
- * endEmbeddedDocument(ContentHandler, Metadata)}
- * or {@link org.apache.tika.sax.RecursiveParserWrapperHandler#
- * endEmbeddedDocument(ContentHandler, Metadata)}.
- * If the handler does not implement {@link java.io.Serializable} or if there is a
- * {@link java.io.NotSerializableException} thrown during serialization, the server will
- * call {@link ContentHandler#toString()} on the ContentHandler and set that value with the
- * {@link TikaCoreProperties#TIKA_CONTENT} key and then
- * serialize and proxy that data back.
- * </p>
- *
- * @param stream the document stream (input)
- * @param handler handler for the XHTML SAX events (output)
- * @param metadata document metadata (input and output)
- * @param context parse context
- * @throws IOException
- * @throws SAXException
- * @throws TikaException
- */
- public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata,
- ParseContext context) throws IOException, SAXException, TikaException {
- if (tis == null) {
- throw new NullPointerException("null stream");
- }
-
- Throwable t;
-
- boolean alive = false;
- ForkClient client = acquireClient();
- try {
- ContentHandler tee =
- (handler instanceof AbstractRecursiveParserWrapperHandler) ? handler :
- new TeeContentHandler(handler, new MetadataContentHandler(metadata));
-
- t = client.call("parseStream", tis, tee, metadata, context);
- alive = true;
- } catch (TikaException te) {
- // Problem occurred on our side
- alive = true;
- throw te;
- } catch (IOException e) {
- // Problem occurred on the other side
- throw new TikaException("Failed to communicate with a forked parser process." +
- " The process has most likely crashed due to some error" +
- " like running out of memory. A new process will be" +
- " started for the next parsing request.", e);
- } finally {
- releaseClient(client, alive);
- }
-
- if (t instanceof IOException) {
- throw (IOException) t;
- } else if (t instanceof SAXException) {
- throw (SAXException) t;
- } else if (t instanceof TikaException) {
- throw (TikaException) t;
- } else if (t != null) {
- throw new TikaException("Unexpected error in forked server process", t);
- }
- }
-
- public synchronized void close() {
- for (ForkClient client : pool) {
- client.close();
- }
- pool.clear();
- poolSize = 0;
- }
-
- private synchronized ForkClient acquireClient() throws IOException, TikaException {
- while (true) {
- ForkClient client = pool.poll();
-
- // Create a new process if there's room in the pool
- if (client == null && currentlyInUse < poolSize) {
- client = newClient();
- }
-
- // Ping the process, and get rid of it if it's inactive
- if (client != null && !client.ping()) {
- client.close();
- client = null;
- }
-
- if (client != null) {
- currentlyInUse++;
- return client;
- } else if (currentlyInUse >= poolSize) {
- try {
- wait();
- } catch (InterruptedException e) {
- throw new TikaException("Interrupted while waiting for a fork parser", e);
- }
- }
- }
- }
-
- private ForkClient newClient() throws IOException, TikaException {
- TimeoutLimits timeoutLimits = new TimeoutLimits(serverPulseMillis, serverParseTimeoutMillis,
- serverWaitTimeoutMillis);
- if (loader == null && parser == null && tikaBin != null && parserFactoryFactory != null) {
- return new ForkClient(tikaBin, parserFactoryFactory, java, timeoutLimits);
- } else if (loader != null && parser != null && tikaBin == null &&
- parserFactoryFactory == null) {
- return new ForkClient(loader, parser, java, timeoutLimits);
- } else if (loader != null && parser == null && tikaBin != null &&
- parserFactoryFactory != null) {
- return new ForkClient(tikaBin, parserFactoryFactory, loader, java, timeoutLimits);
- } else {
- //TODO: make this more useful
- throw new IllegalStateException("Unexpected combination of state items");
- }
- }
-
- private synchronized void releaseClient(ForkClient client, boolean alive) {
- currentlyInUse--;
- if (currentlyInUse + pool.size() < poolSize && alive) {
- if (maxFilesProcessedPerClient > 0 &&
- client.getFilesProcessed() >= maxFilesProcessedPerClient) {
- client.close();
- } else {
- pool.offer(client);
- }
- notifyAll();
- } else {
- client.close();
- }
- }
-
- /**
- * The amount of time in milliseconds that the server
- * should wait before checking to see if the parse has timed out
- * or if the wait has timed out
- * The default is 5 seconds.
- *
- * @param serverPulseMillis milliseconds to sleep before checking if there has been any activity
- */
- public void setServerPulseMillis(long serverPulseMillis) {
- this.serverPulseMillis = serverPulseMillis;
- }
-
- /**
- * The maximum amount of time allowed for the server to try to parse a file.
- * If more than this time elapses, the server shuts down, and the ForkParser
- * throws an exception.
- *
- * @param serverParseTimeoutMillis
- */
- public void setServerParseTimeoutMillis(long serverParseTimeoutMillis) {
- this.serverParseTimeoutMillis = serverParseTimeoutMillis;
- }
-
- /**
- * The maximum amount of time allowed for the server to wait for a new request to parse
- * a file. The server will shutdown after this amount of time, and a new server will have
- * to be started by a new client.
- *
- * @param serverWaitTimeoutMillis
- */
- public void setServerWaitTimeoutMillis(long serverWaitTimeoutMillis) {
- this.serverWaitTimeoutMillis = serverWaitTimeoutMillis;
- }
-
- /**
- * If there is a slowly building memory leak in one of the parsers,
- * it is useful to set a limit on the number of files processed
- * by a server before it is shutdown and restarted. Default value is -1.
- *
- * @param maxFilesProcessedPerClient maximum number of files that a server can handle
- * before the parser shuts down a client and creates
- * a new process. If set to -1, the server is never restarted
- * because of the number of files handled.
- */
- public void setMaxFilesProcessedPerServer(int maxFilesProcessedPerClient) {
- this.maxFilesProcessedPerClient = maxFilesProcessedPerClient;
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ForkProxy.java b/tika-core/src/main/java/org/apache/tika/fork/ForkProxy.java
deleted file mode 100644
index b10eac8de..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ForkProxy.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.Serializable;
-
-public interface ForkProxy extends Serializable {
-
- void init(DataInputStream input, DataOutputStream output);
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ForkResource.java b/tika-core/src/main/java/org/apache/tika/fork/ForkResource.java
deleted file mode 100644
index 9bbd82bdd..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ForkResource.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-public interface ForkResource {
-
- Throwable process(DataInputStream input, DataOutputStream output) throws IOException;
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ForkServer.java b/tika-core/src/main/java/org/apache/tika/fork/ForkServer.java
deleted file mode 100644
index 1a45e4e1f..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ForkServer.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.NotSerializableException;
-import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URL;
-
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-
-class ForkServer implements Runnable {
-
- public static final byte ERROR = -1;
-
- public static final byte DONE = 0;
-
- public static final byte CALL = 1;
-
- public static final byte PING = 2;
-
- public static final byte RESOURCE = 3;
-
- public static final byte READY = 4;
-
- public static final byte FAILED_TO_START = 5;
-
- public static final byte INIT_PARSER_FACTORY_FACTORY = 6;
- public static final byte INIT_LOADER_PARSER = 7;
- public static final byte INIT_PARSER_FACTORY_FACTORY_LOADER = 8;
- private final Object[] lock = new Object[0];
- /**
- * Input stream for reading from the parent process
- */
- private final DataInputStream input;
- /**
- * Output stream for writing to the parent process
- */
- private final DataOutputStream output;
- private final boolean active = true;
- //milliseconds to sleep before checking to see if there has been any reading/writing
- //If no reading or writing in this time, shutdown the server.
- private long serverPulseMillis = 5000;
- private long serverParserTimeoutMillis = 60000;
- private long serverWaitTimeoutMillis = 60000;
- //can't be class Parser because then you'd
- //have to include that in bootstrap jar (legacy mode)
- private Object parser;
- private ClassLoader classLoader;
- private boolean parsing = false;
- private long since;
- /**
- * Sets up a forked server instance using the given stdin/out
- * communication channel.
- *
- * @param input input stream for reading from the parent process
- * @param output output stream for writing to the parent process
- * @throws IOException if the server instance could not be created
- */
- public ForkServer(InputStream input, OutputStream output, long serverPulseMillis,
- long serverParserTimeoutMillis, long serverWaitTimeoutMillis)
- throws IOException {
- this.input = new DataInputStream(input);
- this.output = new DataOutputStream(output);
- this.serverPulseMillis = serverPulseMillis;
- this.serverParserTimeoutMillis = serverParserTimeoutMillis;
- this.serverWaitTimeoutMillis = serverWaitTimeoutMillis;
- this.parsing = false;
- this.since = System.currentTimeMillis();
- }
-
- /**
- * Starts a forked server process using the standard input and output
- * streams for communication with the parent process. Any attempts by
- * stray code to read from standard input or write to standard output
- * is redirected to avoid interfering with the communication channel.
- *
- * @param args command line arguments, ignored
- * @throws Exception if the server could not be started
- */
- public static void main(String[] args) throws Exception {
- long serverPulseMillis = Long.parseLong(args[0]);
- long serverParseTimeoutMillis = Long.parseLong(args[1]);
- long serverWaitTimeoutMillis = Long.parseLong(args[2]);
-
- URL.setURLStreamHandlerFactory(new MemoryURLStreamHandlerFactory());
-
- ForkServer server =
- new ForkServer(System.in, System.out, serverPulseMillis, serverParseTimeoutMillis,
- serverWaitTimeoutMillis);
- System.setIn(new ByteArrayInputStream(new byte[0]));
- System.setOut(System.err);
-
- Thread watchdog = new Thread(server, "Tika Watchdog");
- watchdog.setDaemon(true);
- watchdog.start();
-
- server.processRequests();
- }
-
- public void run() {
- try {
- while (true) {
- synchronized (lock) {
- long elapsed = System.currentTimeMillis() - since;
- if (parsing && elapsed > serverParserTimeoutMillis) {
- break;
- } else if (!parsing && serverWaitTimeoutMillis > 0 &&
- elapsed > serverWaitTimeoutMillis) {
- break;
- }
- }
- Thread.sleep(serverPulseMillis);
- }
- System.exit(0);
- } catch (InterruptedException e) {
- //swallow
- }
- }
-
- public void processRequests() {
- //initialize
- try {
- initializeParserAndLoader();
- } catch (Throwable t) {
- t.printStackTrace();
- System.err.flush();
- try {
- output.writeByte(FAILED_TO_START);
- output.flush();
- } catch (IOException e) {
- e.printStackTrace();
- System.err.flush();
- }
- return;
- }
- //main loop
- try {
- while (true) {
- int request = input.read();
- if (request == -1) {
- break;
- } else if (request == PING) {
- output.writeByte(PING);
- } else if (request == CALL) {
- call(classLoader, parser);
- } else {
- throw new IllegalStateException("Unexpected request");
- }
- output.flush();
- }
- } catch (Throwable t) {
- t.printStackTrace();
- }
- System.err.flush();
- }
-
- private void initializeParserAndLoader()
- throws IOException, ClassNotFoundException, TikaException, SAXException {
- output.writeByte(READY);
- output.flush();
-
- int configIndex = input.read();
- if (configIndex == -1) {
- throw new TikaException("eof! pipe closed?!");
- }
-
- Object firstObject = readObject(ForkServer.class.getClassLoader());
- switch (configIndex) {
- case INIT_PARSER_FACTORY_FACTORY:
- if (firstObject instanceof ParserFactoryFactory) {
- //the user has submitted a parser factory, but no class loader
- classLoader = ForkServer.class.getClassLoader();
- ParserFactory parserFactory = ((ParserFactoryFactory) firstObject).build();
- parser = new ParserWrapper(parserFactory.build());
- } else {
- throw new IllegalArgumentException(
- "Expecting only one object of class ParserFactoryFactory");
- }
- break;
- case INIT_LOADER_PARSER:
- if (firstObject instanceof ClassLoader) {
- classLoader = (ClassLoader) firstObject;
- Thread.currentThread().setContextClassLoader(classLoader);
- //parser from parent process
- parser = readObject(classLoader);
- } else {
- throw new IllegalArgumentException(
- "Expecting ClassLoader followed by a Parser");
- }
- break;
- case INIT_PARSER_FACTORY_FACTORY_LOADER:
- if (firstObject instanceof ParserFactoryFactory) {
- //the user has submitted a parser factory and a class loader
- ParserFactory parserFactory = ((ParserFactoryFactory) firstObject).build();
- parser = new ParserWrapper(parserFactory.build());
- classLoader = (ClassLoader) readObject(ForkServer.class.getClassLoader());
- Thread.currentThread().setContextClassLoader(classLoader);
- } else {
- throw new IllegalStateException(
- "Expecing ParserFactoryFactory followed by a class loader");
- }
- break;
- }
- output.writeByte(READY);
- output.flush();
- }
-
- private void call(ClassLoader loader, Object object) throws Exception {
- synchronized (lock) {
- parsing = true;
- since = System.currentTimeMillis();
- }
- try {
- Method method = getMethod(object, input.readUTF());
- Object[] args = new Object[method.getParameterTypes().length];
- for (int i = 0; i < args.length; i++) {
- args[i] = readObject(loader);
- }
- try {
- method.invoke(object, args);
- output.write(DONE);
- } catch (InvocationTargetException e) {
- output.write(ERROR);
- // Try to send the underlying Exception itself
- Throwable toSend = e.getCause();
- try {
- ForkObjectInputStream.sendObject(toSend, output);
- } catch (NotSerializableException nse) {
- // Need to build a serializable version of it
- TikaException te = new TikaException(toSend.getMessage());
- te.setStackTrace(toSend.getStackTrace());
- ForkObjectInputStream.sendObject(te, output);
- }
-
- }
- } finally {
- synchronized (lock) {
- parsing = false;
- since = System.currentTimeMillis();
- }
- }
- }
-
- private Method getMethod(Object object, String name) {
- Class<?> klass = object.getClass();
- while (klass != null) {
- for (Class<?> iface : klass.getInterfaces()) {
- for (Method method : iface.getMethods()) {
- if (name.equals(method.getName())) {
- return method;
- }
- }
- }
- klass = klass.getSuperclass();
- }
- return null;
- }
-
- /**
- * Deserializes an object from the given stream. The serialized object
- * is expected to be preceded by a size integer, that is used for reading
- * the entire serialization into a memory before deserializing it.
- *
- * @param loader class loader to be used for loading referenced classes
- * @throws IOException if the object could not be deserialized
- * @throws ClassNotFoundException if a referenced class is not found
- */
- private Object readObject(ClassLoader loader) throws IOException, ClassNotFoundException {
- Object object = ForkObjectInputStream.readObject(input, loader);
- if (object instanceof ForkProxy) {
- ((ForkProxy) object).init(input, output);
- }
-
- // Tell the parent process that we successfully received this object
- output.writeByte(ForkServer.DONE);
- output.flush();
- return object;
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/InputStreamProxy.java b/tika-core/src/main/java/org/apache/tika/fork/InputStreamProxy.java
deleted file mode 100644
index cca9b7471..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/InputStreamProxy.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-class InputStreamProxy extends InputStream implements ForkProxy {
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = 4350939227765568438L;
-
- private final int resource;
-
- private transient DataInputStream input;
-
- private transient DataOutputStream output;
-
- public InputStreamProxy(int resource) {
- this.resource = resource;
- }
-
- public void init(DataInputStream input, DataOutputStream output) {
- this.input = input;
- this.output = output;
- }
-
- @Override
- public int read() throws IOException {
- output.writeByte(ForkServer.RESOURCE);
- output.writeByte(resource);
- output.writeInt(1);
- output.flush();
- int n = input.readInt();
- if (n == 1) {
- return input.readUnsignedByte();
- } else {
- return n;
- }
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- output.writeByte(ForkServer.RESOURCE);
- output.writeByte(resource);
- output.writeInt(len);
- output.flush();
- int n = input.readInt();
- if (n > 0) {
- input.readFully(b, off, n);
- }
- return n;
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/InputStreamResource.java b/tika-core/src/main/java/org/apache/tika/fork/InputStreamResource.java
deleted file mode 100644
index 04ba93cdc..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/InputStreamResource.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-class InputStreamResource implements ForkResource {
-
- private final InputStream stream;
-
- public InputStreamResource(InputStream stream) {
- this.stream = stream;
- }
-
- public Throwable process(DataInputStream input, DataOutputStream output) throws IOException {
- int n = input.readInt();
- byte[] buffer = new byte[n];
- int m;
- try {
- m = stream.read(buffer);
- } catch (IOException e) {
- // returning exception causes deadlock
- // return e;
- e.printStackTrace();
- m = -1;
- }
- output.writeInt(m);
- if (m > 0) {
- output.write(buffer, 0, m);
- }
- output.flush();
- return null;
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLConnection.java b/tika-core/src/main/java/org/apache/tika/fork/MemoryURLConnection.java
deleted file mode 100644
index 74a16878d..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLConnection.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLConnection;
-
-class MemoryURLConnection extends URLConnection {
-
- private final byte[] data;
-
- MemoryURLConnection(URL url, byte[] data) {
- super(url);
- this.data = data;
- }
-
- @Override
- public void connect() {
- }
-
- @Override
- public InputStream getInputStream() {
- return new ByteArrayInputStream(data);
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamHandler.java b/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamHandler.java
deleted file mode 100644
index 6f4b1435e..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamHandler.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.IOException;
-import java.lang.ref.WeakReference;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.net.URLStreamHandler;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-class MemoryURLStreamHandler extends URLStreamHandler {
-
- private static final AtomicInteger counter = new AtomicInteger();
-
- private static final List<MemoryURLStreamRecord> records =
- new LinkedList<>();
-
- public static URL createURL(byte[] data) {
- try {
- int i = counter.incrementAndGet();
- URL url = new URI("tika-in-memory", "localhost", "/" + i).toURL();
-
- MemoryURLStreamRecord record = new MemoryURLStreamRecord();
- record.url = new WeakReference<>(url);
- record.data = data;
- records.add(record);
-
- return url;
- } catch (MalformedURLException | URISyntaxException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- protected URLConnection openConnection(URL u) throws IOException {
- Iterator<MemoryURLStreamRecord> iterator = records.iterator();
- while (iterator.hasNext()) {
- MemoryURLStreamRecord record = iterator.next();
- URL url = record.url.get();
- if (url == null) {
- iterator.remove();
- } else if (url == u) {
- return new MemoryURLConnection(u, record.data);
- }
- }
- throw new IOException("Unknown URL: " + u);
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamHandlerFactory.java b/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamHandlerFactory.java
deleted file mode 100644
index 5f3d818d9..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamHandlerFactory.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.net.URLStreamHandler;
-import java.net.URLStreamHandlerFactory;
-
-class MemoryURLStreamHandlerFactory implements URLStreamHandlerFactory {
-
- public URLStreamHandler createURLStreamHandler(String protocol) {
- if ("tika-in-memory".equals(protocol)) {
- return new MemoryURLStreamHandler();
- } else {
- return null;
- }
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamRecord.java b/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamRecord.java
deleted file mode 100644
index 8a72035fd..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/MemoryURLStreamRecord.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.lang.ref.WeakReference;
-import java.net.URL;
-
-class MemoryURLStreamRecord {
-
- public WeakReference<URL> url;
- public byte[] data;
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/MetadataContentHandler.java b/tika-core/src/main/java/org/apache/tika/fork/MetadataContentHandler.java
deleted file mode 100644
index c1f1f5612..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/MetadataContentHandler.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import org.xml.sax.Attributes;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import org.apache.tika.metadata.Metadata;
-
-class MetadataContentHandler extends DefaultHandler {
-
- private final Metadata metadata;
-
- public MetadataContentHandler(Metadata metadata) {
- this.metadata = metadata;
- }
-
- public void startElement(String uri, String local, String name, Attributes attributes)
- throws SAXException {
- if ("meta".equals(local)) {
- String aname = attributes.getValue("name");
- String content = attributes.getValue("content");
- metadata.add(aname, content);
- }
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ParserFactory.java b/tika-core/src/main/java/org/apache/tika/fork/ParserFactory.java
deleted file mode 100644
index 6e0d95738..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ParserFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.parser.Parser;
-
-public abstract class ParserFactory {
-
- final Map<String, String> args;
-
- public ParserFactory(Map<String, String> args) {
- this.args = args;
- }
-
- public abstract Parser build() throws IOException, SAXException, TikaException;
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ParserFactoryFactory.java b/tika-core/src/main/java/org/apache/tika/fork/ParserFactoryFactory.java
deleted file mode 100644
index 4f7b8f175..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ParserFactoryFactory.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-
-import java.io.Serializable;
-import java.lang.reflect.Constructor;
-import java.util.Map;
-
-import org.apache.tika.exception.TikaException;
-
-/**
- * Lightweight, easily serializable class that contains enough information
- * to build a {@link ParserFactory}
- */
-public class ParserFactoryFactory implements Serializable {
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = 4710974869988895410L;
-
- private final String className;
- private final Map<String, String> args;
-
- public ParserFactoryFactory(String className, Map<String, String> args) {
- this.className = className;
- this.args = args;
- }
-
- public ParserFactory build() throws TikaException {
- try {
- Class<?> clazz = Class.forName(className);
- Constructor<?> con = clazz.getConstructor(Map.class);
- return (ParserFactory) con.newInstance(args);
- } catch (ReflectiveOperationException | IllegalStateException e) {
- throw new TikaException("Couldn't create factory", e);
- }
- }
-
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ParserShim.java b/tika-core/src/main/java/org/apache/tika/fork/ParserShim.java
deleted file mode 100644
index 04d134820..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ParserShim.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-
-//temporary workaround to handle requiring parsers take TikaInputStreams
-public interface ParserShim {
-
- void parseStream(InputStream is, ContentHandler recursiveParserWrapperHandler,
- Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException;
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/ParserWrapper.java b/tika-core/src/main/java/org/apache/tika/fork/ParserWrapper.java
deleted file mode 100644
index e9877e849..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/ParserWrapper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.ParserDecorator;
-
-public class ParserWrapper extends ParserDecorator implements ParserShim {
-
- public ParserWrapper(Parser parser) {
- super(parser);
- }
-
- @Override
- public void parseStream(InputStream is, ContentHandler recursiveParserWrapperHandler, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException {
- Parser p = getWrappedParser();
- try (TikaInputStream tis = TikaInputStream.get(is)) {
- p.parse(tis, recursiveParserWrapperHandler, metadata, context);
- }
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/RPWShim.java b/tika-core/src/main/java/org/apache/tika/fork/RPWShim.java
deleted file mode 100644
index 1bcdbd85d..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/RPWShim.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
-
-public class RPWShim extends RecursiveParserWrapper implements ParserShim {
-
- public RPWShim(Parser wrappedParser) {
- super(wrappedParser);
- }
-
- public void parseStream(InputStream is, ContentHandler recursiveParserWrapperHandler,
- Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException {
- try (TikaInputStream tis = TikaInputStream.get(is)) {
- super.parse(tis, recursiveParserWrapperHandler, metadata, context);
- }
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/RecursiveMetadataContentHandlerProxy.java b/tika-core/src/main/java/org/apache/tika/fork/RecursiveMetadataContentHandlerProxy.java
deleted file mode 100644
index 1195ef20d..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/RecursiveMetadataContentHandlerProxy.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.NotSerializableException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.sax.ContentHandlerFactory;
-import org.apache.tika.sax.RecursiveParserWrapperHandler;
-
-/**
- * <p>This class calls #toString() on the ContentHandler, inserts it into the Metadata object
- * and serializes the Metadata object.
- * </p>
- * Ideally, this would serialize the ContentHandler and the Metadata object as separate objects,
- * but we can't guarantee that the ContentHandler is Serializable (e.g. the StringWriter in
- * the WriteOutContentHandler).
- */
-class RecursiveMetadataContentHandlerProxy extends RecursiveParserWrapperHandler
- implements ForkProxy {
-
- public static final byte EMBEDDED_DOCUMENT = 1;
- public static final byte MAIN_DOCUMENT = 2;
- public static final byte HANDLER_AND_METADATA = 3;
- public static final byte METADATA_ONLY = 4;
- public static final byte COMPLETE = 5;
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = 737511106054617524L;
-
- private final int resource;
-
- private transient DataOutputStream output;
-
- public RecursiveMetadataContentHandlerProxy(int resource,
- ContentHandlerFactory contentHandlerFactory) {
- super(contentHandlerFactory);
- this.resource = resource;
- }
-
- public void init(DataInputStream input, DataOutputStream output) {
- this.output = output;
- }
-
- @Override
- public void endEmbeddedDocument(ContentHandler contentHandler, Metadata metadata)
- throws SAXException {
- proxyBackToClient(EMBEDDED_DOCUMENT, contentHandler, metadata);
- decrementEmbeddedDepth();
- }
-
- @Override
- public void endDocument(ContentHandler contentHandler, Metadata metadata) throws SAXException {
- if (hasHitMaximumEmbeddedResources()) {
- metadata.set(EMBEDDED_RESOURCE_LIMIT_REACHED, "true");
- }
- proxyBackToClient(MAIN_DOCUMENT, contentHandler, metadata);
- }
-
- private void proxyBackToClient(int embeddedOrMainDocument, ContentHandler contentHandler,
- Metadata metadata) throws SAXException {
- try {
- output.write(ForkServer.RESOURCE);
- output.writeByte(resource);
- output.writeByte(embeddedOrMainDocument);
- boolean success = false;
- if (contentHandler instanceof Serializable) {
- byte[] bytes = null;
- try {
- bytes = serialize(contentHandler);
- success = true;
- } catch (NotSerializableException e) {
- //object lied
- }
- if (success) {
-
- output.write(HANDLER_AND_METADATA);
- sendBytes(bytes);
- send(metadata);
- output.writeByte(COMPLETE);
- return;
- }
- }
- //if contenthandler is not allegedly or actually Serializable
- //fall back to adding contentHandler.toString() to the metadata object
- //and send that.
- metadata.set(TikaCoreProperties.TIKA_CONTENT, contentHandler.toString());
- output.writeByte(METADATA_ONLY);
- send(metadata);
- output.writeByte(COMPLETE);
- } catch (IOException e) {
- throw new SAXException(e);
- } finally {
- doneSending();
- }
- }
-
- private void send(Object object) throws IOException {
- byte[] bytes = serialize(object);
- sendBytes(bytes);
- }
-
- private void sendBytes(byte[] bytes) throws IOException {
- output.writeInt(bytes.length);
- output.write(bytes);
- output.flush();
- }
-
- private byte[] serialize(Object object) throws IOException {
- //can't figure out why I'm getting an IllegalAccessException
- //when I try to use ForkedObjectInputStream, but
- //not when I do this manually ?!
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- try (ObjectOutputStream oos = new ObjectOutputStream(bos)) {
- oos.writeObject(object);
- oos.flush();
- }
- return bos.toByteArray();
-
- }
-
- private void doneSending() throws SAXException {
- try {
- output.flush();
- } catch (IOException e) {
- throw new SAXException("Unexpected fork proxy problem", e);
- }
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/RecursiveMetadataContentHandlerResource.java b/tika-core/src/main/java/org/apache/tika/fork/RecursiveMetadataContentHandlerResource.java
deleted file mode 100644
index 638e24daa..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/RecursiveMetadataContentHandlerResource.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
-import org.apache.tika.sax.RecursiveParserWrapperHandler;
-
-class RecursiveMetadataContentHandlerResource implements ForkResource {
-
- private static final ContentHandler DEFAULT_HANDLER = new DefaultHandler();
- private final AbstractRecursiveParserWrapperHandler handler;
-
- public RecursiveMetadataContentHandlerResource(RecursiveParserWrapperHandler handler) {
- this.handler = handler;
- }
-
- public Throwable process(DataInputStream input, DataOutputStream output) throws IOException {
- try {
- internalProcess(input);
- return null;
- } catch (SAXException e) {
- return e;
- }
- }
-
- private void internalProcess(DataInputStream input) throws IOException, SAXException {
- byte embeddedOrMain = input.readByte();
- byte handlerAndMetadataOrMetadataOnly = input.readByte();
-
- ContentHandler localContentHandler = DEFAULT_HANDLER;
- if (handlerAndMetadataOrMetadataOnly ==
- RecursiveMetadataContentHandlerProxy.HANDLER_AND_METADATA) {
- localContentHandler = (ContentHandler) readObject(input);
- } else if (handlerAndMetadataOrMetadataOnly !=
- RecursiveMetadataContentHandlerProxy.METADATA_ONLY) {
- throw new IllegalArgumentException(
- "Expected HANDLER_AND_METADATA or METADATA_ONLY, but got:" +
- handlerAndMetadataOrMetadataOnly);
- }
-
- Metadata metadata = (Metadata) readObject(input);
- if (embeddedOrMain == RecursiveMetadataContentHandlerProxy.EMBEDDED_DOCUMENT) {
- handler.endEmbeddedDocument(localContentHandler, metadata);
- } else if (embeddedOrMain == RecursiveMetadataContentHandlerProxy.MAIN_DOCUMENT) {
- handler.endDocument(localContentHandler, metadata);
- } else {
- throw new IllegalArgumentException(
- "Expected either 0x01 or 0x02, but got: " + embeddedOrMain);
- }
- byte isComplete = input.readByte();
- if (isComplete != RecursiveMetadataContentHandlerProxy.COMPLETE) {
- throw new IOException("Expected the 'complete' signal, but got: " + isComplete);
- }
- }
-
- private Object readObject(DataInputStream inputStream) throws IOException {
- try {
- return ForkObjectInputStream.readObject(inputStream, this.getClass().getClassLoader());
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
-
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/TimeoutLimits.java b/tika-core/src/main/java/org/apache/tika/fork/TimeoutLimits.java
deleted file mode 100644
index 6610437c8..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/TimeoutLimits.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-class TimeoutLimits {
-
- private final long pulseMS;
- private final long parseTimeoutMS;
- private final long waitTimeoutMS;
-
-
- TimeoutLimits(long pulseMS, long parseTimeoutMS, long waitTimeoutMS) {
- this.pulseMS = pulseMS;
- this.parseTimeoutMS = parseTimeoutMS;
- this.waitTimeoutMS = waitTimeoutMS;
- }
-
- public long getPulseMS() {
- return pulseMS;
- }
-
- public long getParseTimeoutMS() {
- return parseTimeoutMS;
- }
-
- public long getWaitTimeoutMS() {
- return waitTimeoutMS;
- }
-}
diff --git a/tika-core/src/main/java/org/apache/tika/fork/package-info.java b/tika-core/src/main/java/org/apache/tika/fork/package-info.java
deleted file mode 100644
index 74cdd062d..000000000
--- a/tika-core/src/main/java/org/apache/tika/fork/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Forked parser.
- */
-@aQute.bnd.annotation.Version("1.0.0")
-package org.apache.tika.fork;
diff --git a/tika-core/src/test/java/org/apache/tika/fork/ForkParserTest.java b/tika-core/src/test/java/org/apache/tika/fork/ForkParserTest.java
deleted file mode 100644
index fa3c59723..000000000
--- a/tika-core/src/test/java/org/apache/tika/fork/ForkParserTest.java
+++ /dev/null
@@ -1,665 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.io.Reader;
-import java.io.Serializable;
-import java.io.StringWriter;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
-import org.apache.tika.parser.mock.MockParser;
-import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
-import org.apache.tika.sax.BasicContentHandlerFactory;
-import org.apache.tika.sax.BodyContentHandler;
-import org.apache.tika.sax.ContentHandlerFactory;
-import org.apache.tika.sax.RecursiveParserWrapperHandler;
-
-public class ForkParserTest extends TikaTest {
-
- @TempDir
- Path tempDir;
-
- @Test
- public void testHelloWorld() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserTest.class.getClassLoader(),
- new ForkTestParser())) {
- Metadata metadata = new Metadata();
- ContentHandler output = new BodyContentHandler();
- TikaInputStream tis = TikaInputStream.get(new byte[0]);
- ParseContext context = new ParseContext();
- parser.parse(tis, output, metadata, context);
- assertEquals("Hello, World!", output.toString().trim());
- assertEquals("text/plain", metadata.get(Metadata.CONTENT_TYPE));
- }
- }
-
- @Test
- public void testSerialParsing() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserTest.class.getClassLoader(),
- new ForkTestParser())) {
- ParseContext context = new ParseContext();
- for (int i = 0; i < 10; i++) {
- ContentHandler output = new BodyContentHandler();
- TikaInputStream tis = TikaInputStream.get(new byte[0]);
- parser.parse(tis, output, new Metadata(), context);
- assertEquals("Hello, World!", output.toString().trim());
- }
- }
- }
-
- @Test
- public void testParallelParsing() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserTest.class.getClassLoader(),
- new ForkTestParser())) {
- final ParseContext context = new ParseContext();
-
- Thread[] threads = new Thread[10];
- ContentHandler[] output = new ContentHandler[threads.length];
- for (int i = 0; i < threads.length; i++) {
- final ContentHandler o = new BodyContentHandler();
- output[i] = o;
- threads[i] = new Thread(() -> {
- try {
- TikaInputStream tis = TikaInputStream.get(new byte[0]);
- parser.parse(tis, o, new Metadata(), context);
- } catch (Exception e) {
- e.printStackTrace();
- }
- });
- threads[i].start();
- }
-
- for (int i = 0; i < threads.length; i++) {
- threads[i].join();
- assertEquals("Hello, World!", output[i].toString().trim());
- }
- }
- }
-
- @Test
- public void testPoolSizeReached() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserTest.class.getClassLoader(),
- new ForkTestParser())) {
- final Semaphore barrier = new Semaphore(0);
-
- Thread[] threads = new Thread[parser.getPoolSize()];
- PipedOutputStream[] pipes = new PipedOutputStream[threads.length];
- final ParseContext context = new ParseContext();
- for (int i = 0; i < threads.length; i++) {
- final PipedInputStream input = new PipedInputStream() {
- @Override
- public synchronized int read() throws IOException {
- barrier.release();
- return super.read();
- }
- };
- pipes[i] = new PipedOutputStream(input);
- threads[i] = new Thread(() -> {
- try {
- ContentHandler o = new DefaultHandler();
- try (TikaInputStream tis = TikaInputStream.get(input)) {
- parser.parse(tis, o, new Metadata(), context);
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- });
- threads[i].start();
- }
-
- // Wait until all the background parsers have been started
- barrier.acquire(parser.getPoolSize());
-
- final ContentHandler o = new BodyContentHandler();
- Thread blocked = new Thread(() -> {
- try {
- barrier.release();
- try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
- parser.parse(tis, o, new Metadata(), context);
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- });
- blocked.start();
-
- // Wait until the last thread is started, and then some to
- // make sure that it would have had a chance to start processing
- // data had it not been blocked.
- barrier.acquire();
- Thread.sleep(1000);
-
- assertEquals("", o.toString());
-
- for (int i = 0; i < threads.length; i++) {
- pipes[i].close();
- threads[i].join();
- }
-
- blocked.join();
- assertEquals("Hello, World!", o.toString().trim());
- }
- }
-
- @Test
- public void testPulseAndTimeouts() throws Exception {
-
- ForkParser forkParser =
- new ForkParser(ForkParserTest.class.getClassLoader(), new MockParser());
- forkParser.setServerPulseMillis(500);
- forkParser.setServerParseTimeoutMillis(5000);
- forkParser.setServerWaitTimeoutMillis(60000);
- String sleepCommand = "<mock>\n" + " <write element=\"p\">Hello, World!</write>\n" +
- " <hang millis=\"11000\" heavy=\"false\" interruptible=\"false\" />\n" +
- "</mock>";
- ContentHandler o = new BodyContentHandler(-1);
- Metadata m = new Metadata();
- ParseContext c = new ParseContext();
- try {
- forkParser
- .parse(TikaInputStream.get(sleepCommand.getBytes(StandardCharsets.UTF_8)),
- o, m, c);
- fail("should have thrown IOException");
- } catch (TikaException e) {
- //failed to communicate with forked parser process"
- } finally {
- forkParser.close();
- }
-
- //test setting very short pulse (10 ms) and a parser that takes at least 1000 ms
- forkParser = new ForkParser(ForkParserTest.class.getClassLoader(), new MockParser());
- forkParser.setServerPulseMillis(10);
- forkParser.setServerParseTimeoutMillis(100);
- sleepCommand = "<mock>\n" + " <write element=\"p\">Hello, World!</write>\n" +
- " <hang millis=\"1000\" heavy=\"false\" interruptible=\"false\" />\n" +
- "</mock>";
- o = new BodyContentHandler(-1);
- m = new Metadata();
- c = new ParseContext();
- try {
- forkParser
- .parse(TikaInputStream.get(sleepCommand.getBytes(StandardCharsets.UTF_8)),
- o, m, c);
- fail("Should have thrown exception");
- } catch (IOException | TikaException e) {
- //"should have thrown IOException lost connection"
- } finally {
- forkParser.close();
- }
- }
-
- @Test
- public void testPackageCanBeAccessed() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserTest.class.getClassLoader(),
- new ForkTestParser.ForkTestParserAccessingPackage())) {
- Metadata metadata = new Metadata();
- ContentHandler output = new BodyContentHandler();
- ParseContext context = new ParseContext();
- try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
- parser.parse(tis, output, metadata, context);
- }
- assertEquals("Hello, World!", output.toString().trim());
- assertEquals("text/plain", metadata.get(Metadata.CONTENT_TYPE));
- }
- }
-
- @Test
- public void testRecursiveParserWrapper() throws Exception {
- Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
- new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT,
- 20000));
- try (ForkParser fork = new ForkParser(ForkParserTest.class.getClassLoader(), wrapper);
- TikaInputStream tis = TikaInputStream.get(getResourceAsStream("/test-documents/basic_embedded.xml"))) {
- Metadata metadata = new Metadata();
- ParseContext context = new ParseContext();
- fork.parse(tis, handler, metadata, context);
- }
- List<Metadata> metadataList = handler.getMetadataList();
- Metadata m0 = metadataList.get(0);
- assertEquals("Nikolai Lobachevsky", m0.get(TikaCoreProperties.CREATOR));
- assertContains("main_content", m0.get(TikaCoreProperties.TIKA_CONTENT));
- assertContains("embed1.xml", m0.get(TikaCoreProperties.TIKA_CONTENT));
-
- Metadata m1 = metadataList.get(1);
- assertEquals("embeddedAuthor", m1.get(TikaCoreProperties.CREATOR));
- assertContains("some_embedded_content", m1.get(TikaCoreProperties.TIKA_CONTENT));
- assertEquals("/embed1.xml", m1.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
- }
-
- @Test
- public void testRecursiveParserWrapperMassiveEmbedded() throws Exception {
- Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
- new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT,
- 20000));
- try (ForkParser fork = new ForkParser(ForkParserTest.class.getClassLoader(), wrapper);
- TikaInputStream tis = TikaInputStream.get(getResourceAsStream("/test-documents/massive_embedded.xml"))) {
- Metadata metadata = new Metadata();
- ParseContext context = new ParseContext();
- fork.parse(tis, handler, metadata, context);
- }
- List<Metadata> metadataList = handler.getMetadataList();
- assertEquals(111, metadataList.size());
- Metadata m0 = metadataList.get(0);
- assertEquals("Nikolai Lobachevsky", m0.get(TikaCoreProperties.CREATOR));
- assertContains("main_content", m0.get(TikaCoreProperties.TIKA_CONTENT));
-
- for (int i = 1; i <= 110; i++) {
- assertContains("embed" + i + ".xml", m0.get(TikaCoreProperties.TIKA_CONTENT));
-
- Metadata m1 = metadataList.get(i);
- assertEquals("embeddedAuthor", m1.get(TikaCoreProperties.CREATOR));
- assertContains("some_embedded_content", m1.get(TikaCoreProperties.TIKA_CONTENT));
- assertEquals("/embed" + i + ".xml", m1.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
- }
- }
-
- @Test
- public void testRPWWithEmbeddedNPE() throws Exception {
- Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
- new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT,
- 20000));
- try (ForkParser fork = new ForkParser(ForkParserTest.class.getClassLoader(), wrapper);
- TikaInputStream tis = TikaInputStream.get(getResourceAsStream("/test-documents/embedded_with_npe.xml"))) {
- Metadata metadata = new Metadata();
- ParseContext context = new ParseContext();
- fork.parse(tis, handler, metadata, context);
- }
- List<Metadata> metadataList = handler.getMetadataList();
- Metadata m0 = metadataList.get(0);
- assertEquals("Nikolai Lobachevsky", m0.get(TikaCoreProperties.CREATOR));
- assertContains("main_content", m0.get(TikaCoreProperties.TIKA_CONTENT));
- assertContains("embed1.xml", m0.get(TikaCoreProperties.TIKA_CONTENT));
-
- Metadata m1 = metadataList.get(1);
- assertEquals("embeddedAuthor", m1.get(TikaCoreProperties.CREATOR));
- assertContains("some_embedded_content", m1.get(TikaCoreProperties.TIKA_CONTENT));
- assertEquals("/embed1.xml", m1.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
- assertContains("another null pointer exception",
- m1.get(TikaCoreProperties.EMBEDDED_EXCEPTION));
- }
-
- @Test
- public void testRPWWithMainDocNPE() throws Exception {
- Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
- new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT,
- 20000));
- try (ForkParser fork = new ForkParser(ForkParserTest.class.getClassLoader(), wrapper);
- TikaInputStream tis = TikaInputStream.get(
- getResourceAsStream("/test-documents/embedded_then_npe.xml"))) {
- Metadata metadata = new Metadata();
- ParseContext context = new ParseContext();
- fork.parse(tis, handler, metadata, context);
- fail();
- } catch (TikaException e) {
- assertTrue(e.getCause() instanceof NullPointerException);
- assertContains("another", e.getCause().getMessage());
- }
- List<Metadata> metadataList = handler.getMetadataList();
- Metadata m0 = metadataList.get(0);
- assertEquals("Nikolai Lobachevsky", m0.get(TikaCoreProperties.CREATOR));
- assertContains("main_content", m0.get(TikaCoreProperties.TIKA_CONTENT));
- assertContains("embed1.xml", m0.get(TikaCoreProperties.TIKA_CONTENT));
-
- Metadata m1 = metadataList.get(1);
- assertEquals("embeddedAuthor", m1.get(TikaCoreProperties.CREATOR));
- assertContains("some_embedded_content", m1.get(TikaCoreProperties.TIKA_CONTENT));
- assertEquals("/embed1.xml", m1.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
- }
-
- @Test
- public void testToFileHandler() throws Exception {
- //test that a server-side write-to-file works without proxying back the
- //AbstractContentHandlerFactory
- Path target = Files.createTempFile(tempDir, "fork-to-file-handler-", ".txt");
- try (TikaInputStream tis = getResourceAsStream("/test-documents/basic_embedded.xml")) {
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(new AutoDetectParser());
- ToFileHandler toFileHandler =
- new ToFileHandler(new SBContentHandlerFactory(), target.toFile());
- try (ForkParser forkParser = new ForkParser(ForkParserTest.class.getClassLoader(),
- wrapper)) {
- Metadata m = new Metadata();
- ParseContext context = new ParseContext();
- forkParser.parse(tis, toFileHandler, m, context);
- }
- }
-
- String contents = null;
- try (Reader reader = Files.newBufferedReader(target, StandardCharsets.UTF_8)) {
- contents = IOUtils.toString(reader);
- }
- assertContainsCount(TikaCoreProperties.TIKA_PARSED_BY.getName() +
- " : org.apache.tika.parser.DefaultParser", contents, 2);
- assertContainsCount(TikaCoreProperties.TIKA_PARSED_BY.getName() +
- " : org.apache.tika.parser.mock.MockParser", contents, 2);
- assertContains("Nikolai Lobachevsky", contents);
- assertContains("embeddedAuthor", contents);
- assertContains("main_content", contents);
- assertContains("some_embedded_content", contents);
- assertContains("X-TIKA:embedded_resource_path : /embed1.xml", contents);
-
- }
-
- @Test
- public void testRecursiveParserWrapperWithProxyingContentHandlersAndMetadata()
- throws Exception {
- Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- BufferingHandler handler = new BufferingHandler(new SBContentHandlerFactory());
- try (ForkParser fork = new ForkParser(ForkParserTest.class.getClassLoader(), wrapper);
- TikaInputStream tis = TikaInputStream.get(getResourceAsStream("/test-documents/basic_embedded.xml"))) {
- Metadata metadata = new Metadata();
- ParseContext context = new ParseContext();
- fork.parse(tis, handler, metadata, context);
- }
- List<Metadata> metadataList = handler.getMetadataList();
- List<ContentHandler> contentHandlers = handler.getContentHandlers();
- Metadata m0 = metadataList.get(0);
- String content0 = contentHandlers.get(0).toString();
- assertEquals("Nikolai Lobachevsky", m0.get(TikaCoreProperties.CREATOR));
- assertContains("main_content", content0);
- assertContains("embed1.xml", content0);
-
- Metadata m1 = metadataList.get(1);
- String content1 = contentHandlers.get(1).toString();
- assertEquals("embeddedAuthor", m1.get(TikaCoreProperties.CREATOR));
- assertContains("some_embedded_content", content1);
- assertEquals("/embed1.xml", m1.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
- }
-
-
- @Test
- public void testRPWWithNonSerializableContentHandler() throws Exception {
- Parser parser = new AutoDetectParser();
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
- RecursiveParserWrapperHandler handler =
- new RecursiveParserWrapperHandler(new NonSerializableHandlerFactory());
- try (ForkParser fork = new ForkParser(ForkParserTest.class.getClassLoader(), wrapper);
- TikaInputStream tis = TikaInputStream.get(getResourceAsStream("/test-documents/embedded_then_npe.xml"))) {
- Metadata metadata = new Metadata();
- ParseContext context = new ParseContext();
- fork.parse(tis, handler, metadata, context);
- fail();
- } catch (TikaException e) {
- assertTrue(e.getCause() instanceof NullPointerException);
- assertContains("another", e.getCause().getMessage());
- }
- List<Metadata> metadataList = handler.getMetadataList();
- Metadata m0 = metadataList.get(0);
- assertEquals("Nikolai Lobachevsky", m0.get(TikaCoreProperties.CREATOR));
- assertContains("main_content", m0.get(TikaCoreProperties.TIKA_CONTENT));
- assertContains("embed1.xml", m0.get(TikaCoreProperties.TIKA_CONTENT));
-
- Metadata m1 = metadataList.get(1);
- assertEquals("embeddedAuthor", m1.get(TikaCoreProperties.CREATOR));
- assertContains("some_embedded_content", m1.get(TikaCoreProperties.TIKA_CONTENT));
- assertEquals("/embed1.xml", m1.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
- }
-
- @Test
- public void testNoUTFDataFormatException() throws Exception {
- ContentHandlerProxy proxy = new ContentHandlerProxy(0);
- DataOutputStream output = new DataOutputStream(new ByteArrayOutputStream());
- proxy.init(null, output);
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 65536; i++) {
- sb.append(1);
- }
- proxy.skippedEntity(sb.toString());
- }
-
- @Test
- public void testForkParserDoesntPreventShutdown() throws Exception {
- ExecutorService service = Executors.newFixedThreadPool(1);
- CountDownLatch cdl = new CountDownLatch(1);
- service.submit(() -> {
- try (ForkParser parser = new ForkParser(ForkParserTest.class.getClassLoader(),
- new ForkTestParser.ForkTestParserWaiting())) {
- Metadata metadata = new Metadata();
- ContentHandler output = new BodyContentHandler();
- TikaInputStream tis = TikaInputStream.get(new byte[0]);
- ParseContext context = new ParseContext();
- cdl.countDown();
- parser.parse(tis, output, metadata, context);
- // Don't care about output not planning to get this far
- } catch (IOException | SAXException | TikaException e) {
- throw new RuntimeException(e);
- }
- });
- // Wait to make sure submitted runnable is actually running
- boolean await = cdl.await(1, TimeUnit.SECONDS);
- if (!await) {
- // This should never happen but be thorough
- fail("Future never ran so cannot test cancellation");
- }
- // Parse is being called try and shutdown
- Instant requestShutdown = Instant.now();
- service.shutdownNow();
- service.awaitTermination(15, TimeUnit.SECONDS);
- long secondsSinceShutdown = ChronoUnit.SECONDS.between(requestShutdown, Instant.now());
- assertTrue(secondsSinceShutdown < 5, "Should have shutdown the service in less than 5 seconds");
- }
-
-
- //use this to test that the wrapper handler is acted upon by the server but not proxied back
- private static class ToFileHandler extends AbstractRecursiveParserWrapperHandler {
-
- //this needs to be a file because a File is serializable
- private final File file;
- private OutputStream os;
-
- public ToFileHandler(ContentHandlerFactory contentHandlerFactory, File file) {
- super(contentHandlerFactory);
- this.file = file;
- }
-
- @Override
- public void startDocument() throws SAXException {
- try {
- os = Files.newOutputStream(file.toPath());
- } catch (IOException e) {
- throw new SAXException(e);
- }
- }
-
- @Override
- public void endDocument() throws SAXException {
- try {
- os.flush();
- os.close();
- } catch (IOException e) {
- throw new SAXException(e);
- }
- }
-
- @Override
- public void endEmbeddedDocument(ContentHandler contentHandler, Metadata metadata)
- throws SAXException {
- try {
- byte[] bytes = toString(contentHandler, metadata);
- os.write(bytes, 0, bytes.length);
- } catch (IOException e) {
- throw new SAXException(e);
- }
- }
-
- @Override
- public void endDocument(ContentHandler contentHandler, Metadata metadata)
- throws SAXException {
- try {
- byte[] bytes = toString(contentHandler, metadata);
- os.write(bytes, 0, bytes.length);
- } catch (IOException e) {
- throw new SAXException(e);
- }
- }
-
- private byte[] toString(ContentHandler contentHandler, Metadata metadata) {
- StringBuilder sb = new StringBuilder();
- for (String n : metadata.names()) {
- for (String v : metadata.getValues(n)) {
- sb.append(n).append(" : ").append(v).append("\n");
- }
- }
- if (!contentHandler.getClass().equals(DefaultHandler.class)) {
- sb.append("\n");
- sb.append("CONTENT: ").append(contentHandler);
- sb.append("\n\n");
- }
- return sb.toString().getBytes(StandardCharsets.UTF_8);
- }
- }
-
- private static class SBContentHandler extends DefaultHandler implements Serializable {
- StringBuilder sb = new StringBuilder();
-
- @Override
- public void characters(char[] ch, int start, int length) throws SAXException {
- sb.append(ch, start, length);
- sb.append(" ");
- }
-
- @Override
- public String toString() {
- return sb.toString();
- }
- }
-
- private static class SBContentHandlerFactory implements ContentHandlerFactory {
-
- @Override
- public ContentHandler getNewContentHandler() {
- return new SBContentHandler();
- }
-
- @Override
- public ContentHandler getNewContentHandler(OutputStream os, Charset charset) {
- throw new IllegalArgumentException("can't use this option in this test class");
- }
- }
-
- private static class NonSerializableHandlerFactory implements ContentHandlerFactory {
- @Override
- public ContentHandler getNewContentHandler() {
- return new LyingNonSerializableContentHandler();
- }
-
- @Override
- public ContentHandler getNewContentHandler(OutputStream os, Charset charset) {
- throw new IllegalArgumentException("can't use this option in this test class");
- }
- }
-
- private static class LyingNonSerializableContentHandler extends DefaultHandler
- implements Serializable {
- //StringWriter makes this class not actually Serializable
- //as is.
- StringWriter writer = new StringWriter();
-
- @Override
- public void characters(char[] ch, int start, int length) throws SAXException {
- writer.write(ch, start, length);
- }
-
- @Override
- public String toString() {
- return writer.toString();
- }
- }
-
- //use this to test that a handler that extends RecursiveParserWrapperHandler
- //does have both contenthandlers and metadata objects proxied back from the
- //server.
- private static class BufferingHandler extends RecursiveParserWrapperHandler {
- List<ContentHandler> contentHandlers = new ArrayList<>();
-
- public BufferingHandler(ContentHandlerFactory contentHandlerFactory) {
- super(contentHandlerFactory);
- }
-
-
- @Override
- public void endEmbeddedDocument(ContentHandler contentHandler, Metadata metadata)
- throws SAXException {
- contentHandlers.add(contentHandler);
- metadataList.add(metadata);
- }
-
- @Override
- public void endDocument(ContentHandler contentHandler, Metadata metadata)
- throws SAXException {
- contentHandlers.add(0, contentHandler);
- metadataList.add(0, metadata);
- }
-
- public List<ContentHandler> getContentHandlers() {
- return contentHandlers;
- }
-
- @Override
- public List<Metadata> getMetadataList() {
- return metadataList;
- }
-
- }
-}
diff --git a/tika-core/src/test/java/org/apache/tika/fork/ForkParserTikaBinTest.java b/tika-core/src/test/java/org/apache/tika/fork/ForkParserTikaBinTest.java
deleted file mode 100644
index 2f1ac36ce..000000000
--- a/tika-core/src/test/java/org/apache/tika/fork/ForkParserTikaBinTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Predicate;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-
-import com.google.common.reflect.ClassPath;
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.TikaTest;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.sax.ToXMLContentHandler;
-
-public class ForkParserTikaBinTest extends TikaTest {
- private static final String JAR_FILE_NAME = "mock-tika-app.jar";
- private static final Map<String, String> EMPTY_MAP = Collections.emptyMap();
-
- @TempDir
- private static Path JAR_DIR;
- private static Path JAR_FILE;
-
- @BeforeAll
- public static void bootstrapJar() throws Exception {
- JAR_FILE = JAR_DIR.resolve(JAR_FILE_NAME);
-
- try (JarOutputStream jarOs = new JarOutputStream(Files.newOutputStream(JAR_FILE))) {
- ClassLoader loader = ForkServer.class.getClassLoader();
- ClassPath classPath = ClassPath.from(loader);
- addClasses(jarOs, classPath, ci -> ci.getPackageName().startsWith("org.slf4j"));
- addClasses(jarOs, classPath, ci -> ci.getPackageName().startsWith("org.apache.logging"));
- addClasses(jarOs, classPath,
- ci -> ci.getPackageName().startsWith("org.apache.commons.io"));
- //exclude TypeDetectionBenchmark because it is not serializable
- //exclude UpperCasingContentHandler because we want to test that
- //we can serialize it from the parent process into the forked process
- addClasses(jarOs, classPath, ci -> ci.getPackageName().startsWith("org.apache.tika") &&
- (!ci.getName().contains("TypeDetectionBenchmark")) &&
- (!ci.getName().contains("UpperCasingContentHandler")));
-
- try (InputStream input = ForkParserTikaBinTest.class
- .getResourceAsStream("/org/apache/tika/config/TIKA-2653-vowel-parser-ae.xml")) {
- jarOs.putNextEntry(
- new JarEntry("org/apache/tika/fork/TIKA-2653-vowel-parser-ae.xml"));
- IOUtils.copy(input, jarOs);
- }
- try (InputStream input = ForkParserTikaBinTest.class
- .getResourceAsStream("/org/apache/tika/mime/tika-mimetypes.xml")) {
- jarOs.putNextEntry(new JarEntry("org/apache/tika/mime/tika-mimetypes.xml"));
- IOUtils.copy(input, jarOs);
- }
- try (InputStream input = ForkParserTikaBinTest.class
- .getResourceAsStream("/custom-mimetypes.xml")) {
- jarOs.putNextEntry(new JarEntry("custom-mimetypes.xml"));
- IOUtils.copy(input, jarOs);
- }
-
- jarOs.putNextEntry(new JarEntry("META-INF/services/org.apache.tika.parser.Parser"));
- jarOs.write(
- "org.apache.tika.parser.mock.VowelParser\n".getBytes(StandardCharsets.UTF_8));
- }
-
- Path tikaConfigVowelParser = JAR_DIR.resolve("TIKA_2653-iou.xml");
- try (InputStream is = ForkServer.class
- .getResourceAsStream("/org/apache/tika/config/TIKA-2653-vowel-parser-iou.xml");
- OutputStream os = Files.newOutputStream(tikaConfigVowelParser)) {
- IOUtils.copy(is, os);
- }
- }
-
- private static void addClasses(JarOutputStream jarOs, ClassPath classPath,
- Predicate<ClassPath.ClassInfo> predicate) throws IOException {
- for (ClassPath.ClassInfo classInfo : classPath.getAllClasses()) {
- if (predicate.test(classInfo)) {
- jarOs.putNextEntry(new JarEntry(classInfo.getResourceName()));
- classInfo.asByteSource().copyTo(jarOs);
- }
- }
- }
-
- @Test
- public void testExplicitParserFactory() throws Exception {
- XMLResult xmlResult =
- getXML(new ParserFactoryFactory("org.apache.tika.parser.mock.MockParserFactory",
- EMPTY_MAP));
- assertContains("hello world!", xmlResult.xml);
- assertEquals("Nikolai Lobachevsky", xmlResult.metadata.get(TikaCoreProperties.CREATOR));
- }
-
- @Test
- public void testVowelParserAsDefault() throws Exception {
- ParserFactoryFactory pff =
- new ParserFactoryFactory("org.apache.tika.fork.AutoDetectParserFactory",
- EMPTY_MAP);
- XMLResult xmlResult = getXML(pff);
- assertContains("eooeuiooueoeeao", xmlResult.xml);
- assertEquals("Nikolai Lobachevsky", xmlResult.metadata.get(TikaCoreProperties.CREATOR));
- }
-
- @Test
- public void testPFFWithClassLoaderFromParentProcess() throws Exception {
- //The UpperCasingContentHandler is not sent to the bootstrap test jar file in @BeforeClass.
- //this tests that the content handler was loaded from the parent process.
-
- ParserFactoryFactory pff =
- new ParserFactoryFactory("org.apache.tika.fork.AutoDetectParserFactory",
- EMPTY_MAP);
- XMLResult xmlResult =
- getXML(pff, this.getClass().getClassLoader(), new UpperCasingContentHandler());
- assertContains("EOOEUIOOUEOEEAO", xmlResult.xml);
- assertEquals("Nikolai Lobachevsky", xmlResult.metadata.get(TikaCoreProperties.CREATOR));
-
- }
-
- private XMLResult getXML(ParserFactoryFactory pff)
- throws TikaException, SAXException, IOException {
- return getXML(pff, null, null);
- }
-
- private XMLResult getXML(ParserFactoryFactory pff, ClassLoader classloader,
- ContentHandler contentHandler)
- throws TikaException, SAXException, IOException {
-
- List<String> java = new ArrayList<>();
- java.add("java");
- java.add("-Djava.awt.headless=true");
- ForkParser parser = null;
- if (classloader != null) {
- parser = new ForkParser(JAR_DIR, pff, classloader);
- } else {
- parser = new ForkParser(JAR_DIR, pff);
- }
- parser.setJavaCommand(java);
- parser.setServerPulseMillis(10000);
-
- ContentHandler handler =
- (contentHandler == null) ? new ToXMLContentHandler() : contentHandler;
- Metadata m = new Metadata();
- try (TikaInputStream tis = TikaInputStream.get(getResourceAsStream("/test-documents/example.xml"))) {
- parser.parse(tis, handler, m, new ParseContext());
- } finally {
- parser.close();
- }
- return new XMLResult(handler.toString(), m);
- }
-}
diff --git a/tika-core/src/test/java/org/apache/tika/fork/ForkTestParser.java b/tika-core/src/test/java/org/apache/tika/fork/ForkTestParser.java
deleted file mode 100644
index cfccf78e9..000000000
--- a/tika-core/src/test/java/org/apache/tika/fork/ForkTestParser.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.Set;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.fork.unusedpackage.ClassInUnusedPackage;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.AbstractParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.sax.XHTMLContentHandler;
-
-class ForkTestParser extends AbstractParser implements ParserShim {
-
- /**
- * Serial version UID
- */
- private static final long serialVersionUID = -5492269783593452319L;
-
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- return Collections.singleton(MediaType.TEXT_PLAIN);
- }
-
- @Override
- public void parse(TikaInputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
- parseStream(stream, handler, metadata, context);
- }
-
- public void parseStream(InputStream stream, ContentHandler handler, Metadata metadata,
- ParseContext context) throws IOException, SAXException, TikaException {
- stream.read();
-
- metadata.set(Metadata.CONTENT_TYPE, "text/plain");
-
- XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
- xhtml.startDocument();
- char[] ch = "Hello, World!".toCharArray();
- xhtml.characters(ch, 0, ch.length);
- xhtml.endDocument();
- }
-
- static class ForkTestParserAccessingPackage extends ForkTestParser {
- @Override
- public void parseStream(InputStream stream, ContentHandler handler, Metadata metadata,
- ParseContext context) throws IOException, SAXException, TikaException {
- assertNotNull(ClassInUnusedPackage.class.getPackage());
- super.parseStream(stream, handler, metadata, context);
- }
- }
-
- static class ForkTestParserWaiting extends ForkTestParser {
- @Override
- public void parseStream(InputStream stream, ContentHandler handler, Metadata metadata,
- ParseContext context) throws IOException, SAXException, TikaException {
- try {
- Thread.sleep(10_000);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- super.parseStream(stream, handler, metadata, context);
- }
- }
-}
diff --git a/tika-core/src/test/java/org/apache/tika/fork/UpperCasingContentHandler.java b/tika-core/src/test/java/org/apache/tika/fork/UpperCasingContentHandler.java
deleted file mode 100644
index 3ca513f1a..000000000
--- a/tika-core/src/test/java/org/apache/tika/fork/UpperCasingContentHandler.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork;
-
-import java.util.Locale;
-
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-public class UpperCasingContentHandler extends DefaultHandler {
- StringBuilder sb = new StringBuilder();
-
- @Override
- public void characters(char[] ch, int start, int length) throws SAXException {
- String chars = new String(ch, start, length);
- sb.append(chars.toUpperCase(Locale.US));
- }
-
- @Override
- public String toString() {
- return sb.toString();
- }
-
-}
diff --git a/tika-core/src/test/java/org/apache/tika/fork/unusedpackage/ClassInUnusedPackage.java b/tika-core/src/test/java/org/apache/tika/fork/unusedpackage/ClassInUnusedPackage.java
deleted file mode 100644
index 1de4c4549..000000000
--- a/tika-core/src/test/java/org/apache/tika/fork/unusedpackage/ClassInUnusedPackage.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.fork.unusedpackage;
-
-public class ClassInUnusedPackage {
-}
diff --git a/tika-core/src/test/java/org/apache/tika/parser/mock/AutoDetectParserFactory.java b/tika-core/src/test/java/org/apache/tika/parser/mock/AutoDetectParserFactory.java
deleted file mode 100644
index a4d5e3416..000000000
--- a/tika-core/src/test/java/org/apache/tika/parser/mock/AutoDetectParserFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.mock;
-
-import java.util.Map;
-
-import org.apache.tika.fork.ParserFactory;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.Parser;
-
-/**
- * Simple factory that returns a default AutoDetectParser.
- */
-public class AutoDetectParserFactory extends ParserFactory {
-
- public AutoDetectParserFactory(Map<String, String> args) {
- super(args);
- }
-
- @Override
- public Parser build() {
- return new AutoDetectParser();
- }
-}
diff --git a/tika-core/src/test/java/org/apache/tika/parser/mock/MockParserFactory.java b/tika-core/src/test/java/org/apache/tika/parser/mock/MockParserFactory.java
deleted file mode 100644
index 321c57bc8..000000000
--- a/tika-core/src/test/java/org/apache/tika/parser/mock/MockParserFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.mock;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.fork.ParserFactory;
-import org.apache.tika.parser.Parser;
-
-
-public class MockParserFactory extends ParserFactory {
-
- public MockParserFactory(Map<String, String> args) {
- super(args);
- }
-
- @Override
- public Parser build() throws IOException, TikaException {
- return new MockParser();
- }
-}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
index 3e49db3b2..782b5a5ce 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/PackageParser.java
@@ -88,13 +88,13 @@ public class PackageParser extends AbstractEncodingDetectorParser {
//We used to avoid overwriting file types if the file type
//was a specialization of zip/tar. We determined specialization of zip
//via TikaConfig at parse time.
- //However, TIKA-2483 showed that TikaConfig is not serializable
- //and this causes an exception in the ForkParser.
//The following is an inelegant hack, but until we can serialize TikaConfig,
//or dramatically rework the ForkParser to avoid serialization
//of parsers, this is what we have.
//There is at least a test in PackageParserTest that makes sure that we
//keep this list updated.
+ //This is now legacy behavior: TODO figure out if we can go make to using the
+ //mime registry
static final Set<MediaType> PACKAGE_SPECIALIZATIONS = loadPackageSpecializations();
/**
* Serial version UID
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/fork/ForkParserIntegrationTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/fork/ForkParserIntegrationTest.java
deleted file mode 100644
index f6826ac3c..000000000
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/fork/ForkParserIntegrationTest.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.parser.fork;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.fail;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.NotSerializableException;
-import java.net.ServerSocket;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-
-import org.apache.tika.MultiThreadedTikaTest;
-import org.apache.tika.Tika;
-import org.apache.tika.TikaLoaderHelper;
-import org.apache.tika.detect.Detector;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.fork.ForkParser;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.metadata.Metadata;
-import org.apache.tika.mime.MediaType;
-import org.apache.tika.parser.EmptyParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
-import org.apache.tika.parser.microsoft.libpst.LibPstParser;
-import org.apache.tika.sax.BodyContentHandler;
-
-/**
- * Test that the ForkParser correctly behaves when
- * wired in to the regular Parsers and their test data
- */
-public class ForkParserIntegrationTest extends MultiThreadedTikaTest {
-
- private Tika tika = new Tika(); // TODO Use TikaConfig instead, when it works
-
- /**
- * Simple text parsing
- */
- @Test
- public void testForkedTextParsing() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserIntegrationTest.class.getClassLoader(),
- tika.getParser())) {
- ContentHandler output = new BodyContentHandler();
- InputStream stream = getResourceAsStream("/test-documents/testTXT.txt");
- ParseContext context = new ParseContext();
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, output, new Metadata(), context);
- }
-
- String content = output.toString();
- assertContains("Test d'indexation", content);
- assertContains("http://www.apache.org", content);
- }
- }
-
- /**
- * TIKA-831 Parsers throwing errors should be caught and
- * properly reported
- */
- @Test
- public void testParsingErrorInForkedParserShouldBeReported() throws Exception {
- BrokenParser brokenParser = new BrokenParser();
- ForkParser parser = new ForkParser(ForkParser.class.getClassLoader(), brokenParser);
- InputStream stream = getResourceAsStream("/test-documents/testTXT.txt");
-
- // With a serializable error, we'll get that back
- try {
- ContentHandler output = new BodyContentHandler();
- ParseContext context = new ParseContext();
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, output, new Metadata(), context);
- }
- fail("Expected TikaException caused by Error");
- } catch (TikaException e) {
- assertEquals(brokenParser.err, e.getCause());
- } finally {
- parser.close();
- stream.close();
- }
-
- // With a non serializable one, we'll get something else
- // TODO Fix this test
- brokenParser = new BrokenParser();
- brokenParser.re = new WontBeSerializedError("Can't Serialize");
- parser = new ForkParser(ForkParser.class.getClassLoader(), brokenParser);
-// try {
-// ContentHandler output = new BodyContentHandler();
-// ParseContext context = new ParseContext();
-// parser.parse(stream, output, new Metadata(), context);
-// fail("Expected TikaException caused by Error");
-// } catch (TikaException e) {
-// assertEquals(TikaException.class, e.getCause().getClass());
-// assertEquals("Bang!", e.getCause().getMessage());
-// }
- }
-
- /**
- * If we supply a non serializable object on the ParseContext,
- * check we get a helpful exception back
- */
- @Test
- public void testParserHandlingOfNonSerializable() throws Exception {
- ForkParser parser =
- new ForkParser(ForkParserIntegrationTest.class.getClassLoader(), tika.getParser());
-
- ParseContext context = new ParseContext();
- context.set(Detector.class, new Detector() {
- public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) {
- return MediaType.OCTET_STREAM;
- }
- });
-
- try {
- ContentHandler output = new BodyContentHandler();
- InputStream stream = getResourceAsStream("/test-documents/testTXT.txt");
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, output, new Metadata(), context);
- }
- fail("Should have blown up with a non serializable ParseContext");
- } catch (TikaException e) {
- // Check the right details
- assertNotNull(e.getCause());
- assertEquals(NotSerializableException.class, e.getCause().getClass());
- assertEquals("Unable to serialize ParseContext to pass to the Forked Parser",e.getMessage());
- } finally {
- parser.close();
- }
- }
-
- /**
- * TIKA-832
- */
- @Test
- public void testAttachingADebuggerOnTheForkedParserShouldWork() throws Exception {
- ParseContext context = new ParseContext();
- context.set(Parser.class, tika.getParser());
-
- int availSocket = 0;
- try (ServerSocket serverSocket = new ServerSocket(0)) {
- Assertions.assertNotNull(serverSocket);
- availSocket = serverSocket.getLocalPort();
- Assertions.assertTrue(availSocket > 0);
- } catch (IOException e) {
- fail("Port is not available");
- }
- ForkParser parser =
- new ForkParser(ForkParserIntegrationTest.class.getClassLoader(), tika.getParser());
- parser.setJavaCommand(Arrays.asList("java", "-Xmx32m", "-Xdebug", "-Xrunjdwp:transport" +
- "=dt_socket,address=" + availSocket + ",server=y,suspend=n"));
- try {
- ContentHandler body = new BodyContentHandler();
- InputStream stream = getResourceAsStream("/test-documents/testTXT.txt");
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, body, new Metadata(), context);
- }
- String content = body.toString();
- assertContains("Test d'indexation", content);
- assertContains("http://www.apache.org", content);
- } finally {
- parser.close();
- }
- }
-
- /**
- * TIKA-808 - Ensure that parsing of our test PDFs work under
- * the Fork Parser, to ensure that complex parsing behaves
- */
- @Test
- public void testForkedPDFParsing() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserIntegrationTest.class.getClassLoader(),
- tika.getParser())) {
- ContentHandler output = new BodyContentHandler();
- InputStream stream = getResourceAsStream("/test-documents/testPDF.pdf");
- ParseContext context = new ParseContext();
- context.set(Parser.class, new EmptyParser());
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, output, new Metadata(), context);
- }
-
- String content = output.toString();
- assertContains("Apache Tika", content);
- assertContains("Tika - Content Analysis Toolkit", content);
- assertContains("incubator", content);
- assertContains("Apache Software Foundation", content);
- }
- }
-
- @Test
- public void testForkedPackageParsing() throws Exception {
- try (ForkParser parser = new ForkParser(ForkParserIntegrationTest.class.getClassLoader(),
- tika.getParser())) {
- ContentHandler output = new BodyContentHandler();
- InputStream stream = getResourceAsStream("/test-documents/moby.zip");
- ParseContext context = new ParseContext();
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, output, new Metadata(), context);
- }
- assertContains("Moby Dick", output.toString());
- }
- }
-
- @Test
- public void testLibPstParser() throws Exception {
- if (! new LibPstParser().checkQuietly()) {
- return;
- }
- Parser autoDetectParser = TikaLoaderHelper.getLoader("tika-config-lib-pst.json").loadAutoDetectParser();
- try (ForkParser parser = new ForkParser(ForkParserIntegrationTest.class.getClassLoader(),
- autoDetectParser)) {
- ContentHandler output = new BodyContentHandler();
- InputStream stream = getResourceAsStream("/test-documents/testPST.pst");
- ParseContext context = new ParseContext();
- try (TikaInputStream tis = TikaInputStream.get(stream)) {
- parser.parse(tis, output, new Metadata(), context);
- }
- assertContains("Barry Olddog", output.toString());
- }
- }
-
- @Test
- @Disabled("use for development/one off testing. This is a beast and takes enormous " +
- "resources and time")
- public void smokeTest() throws Exception {
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(tika.getParser());
- int numThreads = 5;
- ForkParser parser =
- new ForkParser(ForkParserIntegrationTest.class.getClassLoader(), wrapper);
- parser.setServerPulseMillis(500);
- parser.setServerParseTimeoutMillis(1000);
- parser.setPoolSize(numThreads);
- ParseContext[] parseContexts = new ParseContext[numThreads];
- for (int i = 0; i < numThreads; i++) {
- parseContexts[i] = new ParseContext();
- }
- try {
- super.testMultiThreaded(parser, parseContexts, numThreads, 5, pathname -> {
- return pathname.getAbsolutePath().contains("mock");
- /*
- if (pathname.getName().contains("11_hang.rar") ||
- pathname.getName().contains("radar_profiles_2009.mat") ||
- pathname.getAbsolutePath().contains("mock")) {
- //return false;
- }
- return true;*/
- });
- } catch (Throwable t) {
- t.printStackTrace();
- } finally {
- parser.close();
- }
- }
-
- /**
- * This error has a message and an equals() implementation as to be able
- * to match it against the serialized version of itself.
- */
- static class AnError extends Error {
- private static final long serialVersionUID = -6197267350768803348L;
- private String message;
-
- AnError(String message) {
- super(message);
- this.message = message;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- AnError anError = (AnError) o;
-
- if (!message.equals(anError.message)) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return message.hashCode();
- }
- }
-
- /**
- * This error isn't serializable on the server, so can't be sent back
- * to the Fork Client once it has occured
- */
- static class WontBeSerializedError extends RuntimeException {
- private static final long serialVersionUID = 1L;
-
- WontBeSerializedError(String message) {
- super(message);
- }
-
- private void writeObject(java.io.ObjectOutputStream out) {
- RuntimeException e = new RuntimeException("Bang!");
- boolean found = false;
- for (StackTraceElement ste : e.getStackTrace()) {
- if (ste.getClassName().equals(ForkParser.class.getName())) {
- found = true;
- break;
- }
- }
- if (!found) {
- throw e;
- }
- }
- }
-
- static class BrokenParser implements Parser {
- private static final long serialVersionUID = 995871497930817839L;
- public Error err = new AnError("Simulated fail");
- public RuntimeException re = null;
-
- public Set<MediaType> getSupportedTypes(ParseContext context) {
- return new HashSet<>(Collections.singletonList(MediaType.TEXT_PLAIN));
- }
-
- public void parse(TikaInputStream stream, ContentHandler handler, Metadata metadata,
- ParseContext context) throws IOException, SAXException, TikaException {
- if (re != null) {
- throw re;
- }
- throw err;
- }
- }
-
-}