You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2020/12/18 21:34:13 UTC
[tika] branch main updated: TIKA-3247 -- make -spawnChild the
default for tika-server in Tika 2.0
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new a5772b9 TIKA-3247 -- make -spawnChild the default for tika-server in Tika 2.0
a5772b9 is described below
commit a5772b96011779ac675e0061ffea617dc0a5077d
Author: tallison <ta...@apache.org>
AuthorDate: Fri Dec 18 16:29:11 2020 -0500
TIKA-3247 -- make -spawnChild the default for tika-server in Tika 2.0
---
CHANGES.txt | 41 +++--
.../apache/tika/cli/BatchCommandLineBuilder.java | 6 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 6 +-
.../java/org/apache/tika/batch/BatchProcess.java | 2 +-
.../apache/tika/batch/BatchProcessDriverCLI.java | 26 +--
.../java/org/apache/tika/batch/Interrupter.java | 2 +-
.../batch/fs/strawman/StrawManTikaAppDriver.java | 4 +-
.../org/apache/tika/batch/fs/FSBatchTestBase.java | 2 +-
.../java/org/apache/tika/fuzzing/cli/FuzzOne.java | 2 +-
.../apache/tika/fuzzing/cli/FuzzingCLIConfig.java | 4 +-
.../apache/tika/parser/mock/MockParserTest.java | 2 +-
.../bin/install_tika_service.sh | 6 +-
tika-server/tika-server-classic/bin/tika | 24 ++-
tika-server/tika-server-classic/bin/tika.in.sh | 9 +-
.../tika/server/core/ServerStatusWatcher.java | 18 +-
.../apache/tika/server/core/ServerTimeouts.java | 44 ++---
.../org/apache/tika/server/core/TikaServerCli.java | 113 +++++++------
.../tika/server/core/TikaServerWatchDog.java | 185 ++++++++++-----------
.../server/core/TikaServerIntegrationTest.java | 35 ++--
.../logging/{log4j_child.xml => log4j_forked.xml} | 0
20 files changed, 273 insertions(+), 258 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 0d61915..577c438 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,21 +1,36 @@
Release 2.0.0 - ???
BREAKING CHANGES in 2.0.0
+ * General
+ * Remove deprecated Metadata keys/properties (TIKA-1974).
+ * Removed dangerous calls to read an inputstream or convert to bytes
+ without specifying a charset
+
+ * tika-parsers
+ * The parser modules have been broken into three main modules:
+ tika-parsers-classic, tika-parsers-extended and tika-parsers-advanced.
+ Users may now need to add tika-parsers-extended to tika-app and
+ tika-server to include parsers that used to be included by default
+ (for example: envi, gdal, grib, isatab, netcdf).
+ * ChmParser was moved to org.apache.tika.parser.microsoft.chm
+ * RTFParser was moved to org.apache.tika.parser.microsoft.rtf
+
+ * tika-app
+
+ * tika-server
+ * tika-server now by default forks a process to isolate the parsing
+ in the forked process (this was called the -spawnChild option
+ in tika-1.x). Clients must now expect that tika-server
+ will restart on OOM, timeouts, crashes or after parsing a
+ large number of files. When this happens tika-server will restand and not
+ receive connections for brief periods. The less robust, legacy behavior
+ of not forking a process is available with "-noFork"
+
+ * tika-server's /metadata endpoint requires tika-server-classic to write XMP/rdf output.
+ This output is not available in tika-server-core.
- * ChmParser was moved to org.apache.tika.parser.microsoft.chm
-
- * RTFParser was moved to org.apache.tika.parser.microsoft.rtf
-
- * Remove deprecated Metadata keys/properties (TIKA-1974).
-
- * Removed dangerous calls to read an inputstream or convert to bytes without specifying a charset
-
+ Other changes
* General code cleanup (PeterAlfredLee)
- * tika-server's /metadata endpoint requires tika-server-classic to write XMP/rdf output.
- This output is not available in tika-server-core.
-
- Other changes
-
Release 1.26 - ???
* Great optimization in ForkParser (TIKA-3237).
diff --git a/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java b/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java
index 15bdf40..00fa86f 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java
@@ -48,7 +48,7 @@ class BatchCommandLineBuilder {
Map<String, String> processArgs = new LinkedHashMap<String, String>();
Map<String, String> jvmOpts = new LinkedHashMap<String,String>();
//take the args, and divide them into process args and options for
- //the child jvm process (i.e. log files, etc)
+ //the forked jvm process (i.e. log files, etc)
mapifyArgs(args, processArgs, jvmOpts);
//now modify processArgs in place
@@ -114,10 +114,10 @@ class BatchCommandLineBuilder {
/**
* Take the input args and separate them into args that belong on the commandline
- * and those that belong as jvm args for the child process.
+ * and those that belong as jvm args for the forked process.
* @param args -- literal args from TikaCLI commandline
* @param commandLine args that should be part of the batch commandline
- * @param jvmArgs args that belong as jvm arguments for the child process
+ * @param jvmArgs args that belong as jvm arguments for the forked process
*/
private static void mapifyArgs(final String[] args,
final Map<String, String> commandLine,
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index b246207..2849f1a 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -642,9 +642,9 @@ public class TikaCLI {
out.println(" -numConsumers Number of processing threads");
out.println(" -bc Batch config file");
out.println(" -maxRestarts Maximum number of times the ");
- out.println(" watchdog process will restart the child process.");
+ out.println(" watchdog process will restart the forked process.");
out.println(" -timeoutThresholdMillis Number of milliseconds allowed to a parse");
- out.println(" before the process is killed and restarted");
+ out.println(" before the process is terminated and restarted");
out.println(" -fileList List of files to process, with");
out.println(" paths relative to the input directory");
out.println(" -includeFilePat Regular expression to determine which");
@@ -655,7 +655,7 @@ public class TikaCLI {
out.println();
out.println(" Control the type of output with -x, -h, -t and/or -J.");
out.println();
- out.println(" To modify child process jvm args, prepend \"J\" as in:");
+ out.println(" To modify forked process jvm args, prepend \"J\" as in:");
out.println(" -JXmx4g or -JDlog4j.configuration=file:log4j.xml.");
}
diff --git a/tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java b/tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java
index 27708f4..38b8954 100644
--- a/tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java
+++ b/tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java
@@ -516,7 +516,7 @@ public class BatchProcess implements Callable<ParallelFileProcessingResult> {
/**
* If there is an early termination via an interrupt or too many timed out consumers
* or because a consumer or other Runnable threw a Throwable, pause this long
- * before killing the consumers and other threads.
+ * before interrupting the consumers and other threads.
*
* Typically makes sense for this to be the same or slightly larger than
* timeoutThresholdMillis
diff --git a/tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java b/tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java
index d570f72..35cfc08 100644
--- a/tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java
+++ b/tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java
@@ -54,7 +54,7 @@ public class BatchProcessDriverCLI {
//how many times to wait pulseMillis milliseconds if a restart
//message has been received through stdout, but the
- //child process has not yet exited
+ //forked process has not yet exited
private int waitNumLoopsAfterRestartMessage = 60;
private int loopsAfterRestartMessageReceived = 0;
@@ -75,7 +75,7 @@ public class BatchProcessDriverCLI {
private final String[] commandLine;
private int numRestarts = 0;
- private boolean redirectChildProcessToStdOut = true;
+ private boolean redirectForkedProcessToStdOut = true;
public BatchProcessDriverCLI(String[] commandLine){
this.commandLine = tryToReadMaxRestarts(commandLine);
@@ -114,7 +114,7 @@ public class BatchProcessDriverCLI {
try {
LOG.trace("about to check exit value");
exit = process.exitValue();
- LOG.info("The child process has finished with an exit value of: {}", exit);
+ LOG.info("The forked process has finished with an exit value of: {}", exit);
stop();
} catch (IllegalThreadStateException e) {
//hasn't exited
@@ -194,7 +194,7 @@ public class BatchProcessDriverCLI {
}
}
LOG.error("Process didn't stop after 60 seconds after shutdown. " +
- "I am forcefully killing it.");
+ "I am forcefully terminating it.");
}
interruptWatcherThread.interrupt();
}
@@ -208,7 +208,7 @@ public class BatchProcessDriverCLI {
}
/**
- * Tries to restart (stop and then start) the child process
+ * Tries to restart (stop and then start) the forked process
* @return whether or not this was successful, will be false if numRestarts >= maxProcessRestarts
* @throws Exception
*/
@@ -264,11 +264,11 @@ public class BatchProcessDriverCLI {
/**
* Typically only used for testing. This determines whether or not
- * to redirect child process's stdOut to driver's stdout
- * @param redirectChildProcessToStdOut should the driver redirect the child's stdout
+ * to redirect forked process's stdOut to driver's stdout
+ * @param redirectForkedProcessToStdOut should the driver redirect the child's stdout
*/
- public void setRedirectChildProcessToStdOut(boolean redirectChildProcessToStdOut) {
- this.redirectChildProcessToStdOut = redirectChildProcessToStdOut;
+ public void setRedirectForkedProcessToStdOut(boolean redirectForkedProcessToStdOut) {
+ this.redirectForkedProcessToStdOut = redirectForkedProcessToStdOut;
}
/**
@@ -298,8 +298,8 @@ public class BatchProcessDriverCLI {
}
/**
- * Class that writes to the child process
- * to force an interrupt in the child process.
+ * Class that writes to the forked process
+ * to force an interrupt in the forked process.
*/
private class InterruptWriter implements Runnable {
private final Writer writer;
@@ -341,7 +341,7 @@ public class BatchProcessDriverCLI {
try {
LOG.trace("gobbler starting to read");
while ((line = reader.readLine()) != null && this.running) {
- if (redirectChildProcessToStdOut) {
+ if (redirectForkedProcessToStdOut) {
System.out.println("BatchProcess:" + line);
}
}
@@ -389,7 +389,7 @@ public class BatchProcessDriverCLI {
public static void main(String[] args) throws Exception {
final BatchProcessDriverCLI runner = new BatchProcessDriverCLI(args);
- //make absolutely certain that the child process is killed
+ //make absolutely certain that the forked process is terminated
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
diff --git a/tika-batch/src/main/java/org/apache/tika/batch/Interrupter.java b/tika-batch/src/main/java/org/apache/tika/batch/Interrupter.java
index eea1308..b148dd5 100644
--- a/tika-batch/src/main/java/org/apache/tika/batch/Interrupter.java
+++ b/tika-batch/src/main/java/org/apache/tika/batch/Interrupter.java
@@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory;
/**
* Class that waits for input on System.in. If this reads
* EOF or if there is an exception from the parent's IO,
- * this will send a signal to shutdown the child process.
+ * this will send a signal to shutdown the forked process.
* <p>
* This will call System.exit(-1) if the process
* doesn't stop after {@link #pauseOnEarlyTermination}
diff --git a/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java b/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
index 7b0bd1e..3d72c12 100644
--- a/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
+++ b/tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
@@ -152,8 +152,8 @@ public class StrawManTikaAppDriver implements Callable<Integer> {
}
}
if (!finished) {
- LOG.warn("Had to kill process working on: {}", file.toAbsolutePath());
- proc.destroy();
+ LOG.warn("Had to terminate process working on: {}", file.toAbsolutePath());
+ proc.destroyForcibly();
}
try {
proc.getOutputStream().flush();
diff --git a/tika-batch/src/test/java/org/apache/tika/batch/fs/FSBatchTestBase.java b/tika-batch/src/test/java/org/apache/tika/batch/fs/FSBatchTestBase.java
index 5ad2f9a..c9e10b3 100644
--- a/tika-batch/src/test/java/org/apache/tika/batch/fs/FSBatchTestBase.java
+++ b/tika-batch/src/test/java/org/apache/tika/batch/fs/FSBatchTestBase.java
@@ -219,7 +219,7 @@ public abstract class FSBatchTestBase extends TikaTest {
commandLine.addAll(Arrays.asList(args));
BatchProcessDriverCLI driver = new BatchProcessDriverCLI(commandLine.toArray(new String[0]));
- driver.setRedirectChildProcessToStdOut(false);
+ driver.setRedirectForkedProcessToStdOut(false);
return driver;
}
diff --git a/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzOne.java b/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzOne.java
index faa1383..98c4438 100644
--- a/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzOne.java
+++ b/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzOne.java
@@ -52,7 +52,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
- * Child process that runs against a single input file
+ * Forked process that runs against a single input file
*/
public class FuzzOne {
private static final Logger LOG = LoggerFactory.getLogger(FuzzOne.class);
diff --git a/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzingCLIConfig.java b/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzingCLIConfig.java
index 206510b..5563307 100644
--- a/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzingCLIConfig.java
+++ b/tika-fuzzing/src/main/java/org/apache/tika/fuzzing/cli/FuzzingCLIConfig.java
@@ -85,7 +85,7 @@ public class FuzzingCLIConfig {
.build())
.addOption(Option.builder("x")
.longOpt("xmx")
- .desc("e.g. 1G, max heap appended to -Xmx in the child process")
+ .desc("e.g. 1G, max heap appended to -Xmx in the forked process")
.hasArg(true)
.required(false)
.build())
@@ -138,7 +138,7 @@ public class FuzzingCLIConfig {
//times to retry a seed file after a catastrophic failure
int retries;
- //xmx for child process, e.g. 512m or 1G
+ //xmx for forked process, e.g. 512m or 1G
String xmx;
Path inputDir;
Path outputDir;
diff --git a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/mock/MockParserTest.java b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/mock/MockParserTest.java
index 486f205..2376032 100644
--- a/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/mock/MockParserTest.java
+++ b/tika-parsers/tika-parsers-classic/tika-parsers-classic-package/src/test/java/org/apache/tika/parser/mock/MockParserTest.java
@@ -135,7 +135,7 @@ public class MockParserTest extends TikaTest {
@Test
public void testRealOOM() throws Exception {
//Note: we're not actually testing the diff between fake and real oom
- //i.e. by creating child process and setting different -Xmx or
+ //i.e. by creating a forked process and setting different -Xmx or
//memory profiling.
Metadata m = new Metadata();
assertThrowable("real_oom.xml", m, OutOfMemoryError.class, "Java heap space");
diff --git a/tika-server/tika-server-classic/bin/install_tika_service.sh b/tika-server/tika-server-classic/bin/install_tika_service.sh
index b414d32..fa9c4e7 100755
--- a/tika-server/tika-server-classic/bin/install_tika_service.sh
+++ b/tika-server/tika-server-classic/bin/install_tika_service.sh
@@ -224,10 +224,6 @@ if [ -z "$TIKA_PORT" ]; then
TIKA_PORT=9998
fi
-if [ -z "$TIKA_SPAWN_CHILD_OPTS" ]; then
- TIKA_SPAWN_CHILD_OPTS="-spawnChild"
-fi
-
if [ -z "$TIKA_UPGRADE" ]; then
TIKA_UPGRADE=NO
fi
@@ -336,7 +332,7 @@ else
LOG4J_PROPS=\"$TIKA_VAR_DIR/log4j.properties\"
TIKA_LOGS_DIR=\"$TIKA_VAR_DIR/logs\"
TIKA_PORT=\"$TIKA_PORT\"
-TIKA_SPAWN_CHILD_OPTS=\"$TIKA_SPAWN_CHILD_OPTS\"
+TIKA_FORKED_OPTS=\"$TIKA_FORKED_OPTS\"
" >> "/etc/default/$TIKA_SERVICE.in.sh"
fi
chown root:${TIKA_USER} "/etc/default/$TIKA_SERVICE.in.sh"
diff --git a/tika-server/tika-server-classic/bin/tika b/tika-server/tika-server-classic/bin/tika
index e8c6e02..9c85deb 100755
--- a/tika-server/tika-server-classic/bin/tika
+++ b/tika-server/tika-server-classic/bin/tika
@@ -207,7 +207,7 @@ function spinner() {
printf " \b\b\b\b"
}
-# uses kill -9 to stop Tika process
+# uses terminate -9 to stop Tika process
function stop_tika() {
DIR="$1"
@@ -217,14 +217,14 @@ function stop_tika() {
echo "ERROR: No PID found for Tika running on port $TIKA_PORT ... script fails."
exit 1
fi
- echo "Sending kill command to Tika running on port $TIKA_PORT with process $TIKA_PID"
+ echo "Sending terminate command to Tika running on port $TIKA_PORT with process $TIKA_PID"
kill -9 $TIKA_PID
rm -f "$TIKA_PID_DIR/tika-$TIKA_PORT.pid"
sleep 1
CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $TIKA_PID | sort -r | tr -d ' '`
if [ "$CHECK_PID" != "" ]; then
- echo "ERROR: Failed to kill previous Tika Java process $TIKA_PID ... script fails."
+ echo "ERROR: Failed to terminate previous Tika Java process $TIKA_PID ... script fails."
exit 1
fi
} # end stop_tika
@@ -352,10 +352,6 @@ if [ $# -gt 0 ]; then
done
fi
-if [ -z "$TIKA_SPAWN_CHILD_OPTS" ]; then
- TIKA_SPAWN_CHILD_OPTS="-spawnChild"
-fi
-
if [ -z "$TIKA_SERVER_DIR" ]; then
TIKA_SERVER_DIR="$DEFAULT_SERVER_DIR"
fi
@@ -467,7 +463,7 @@ function start_tika() {
echo -e " JAVA_MEM_OPTS = ${JAVA_MEM_OPTS[@]}"
echo -e " GC_TUNE = ${GC_TUNE[@]}"
echo -e " GC_LOG_OPTS = ${GC_LOG_OPTS[@]}"
- echo -e " TIKA_SPAWN_CHILD_OPTS = $TIKA_SPAWN_CHILD_OPTS"
+ echo -e " TIKA_FORKED_OPTS = $TIKA_FORKED_OPTS"
if [ "$TIKA_OPTS" != "" ]; then
echo -e " TIKA_OPTS = ${TIKA_OPTS[@]}"
@@ -515,17 +511,17 @@ function start_tika() {
if [ "$run_in_foreground" == "true" ]; then
if $verbose ; then
echo "Startup command"
- echo "$JAVA ${TIKA_START_OPTS[@]} $TIKA_ADDL_ARGS -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_SPAWN_CHILD_OPTS"
+ echo "$JAVA ${TIKA_START_OPTS[@]} $TIKA_ADDL_ARGS -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_FORKED_OPTS"
fi
- exec "$JAVA" "${TIKA_START_OPTS[@]}" $TIKA_ADDL_ARGS -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_SPAWN_CHILD_OPTS
+ exec "$JAVA" "${TIKA_START_OPTS[@]}" $TIKA_ADDL_ARGS -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_TIKA_FORKED_OPTS
else
# run Tika in the background
if $verbose ; then
echo "Startup command"
- echo "$JAVA ${TIKA_START_OPTS[@]} $TIKA_ADDL_ARGS -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_SPAWN_CHILD_OPTS $TIKA_LOGS_DIR/tika-$TIKA_PORT-console.log $TIKA_PID_DIR/tika-$TIKA_PORT.pid"
+ echo "$JAVA ${TIKA_START_OPTS[@]} $TIKA_ADDL_ARGS -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_FORKED_OPTS $TIKA_LOGS_DIR/tika-$TIKA_PORT-console.log $TIKA_PID_DIR/tika-$TIKA_PORT.pid"
fi
nohup "$JAVA" "${TIKA_START_OPTS[@]}" $TIKA_ADDL_ARGS \
- -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_SPAWN_CHILD_OPTS \
+ -jar $TIKA_SERVER_JAR -p $TIKA_PORT -h $TIKA_HOST $TIKA_TIKA_FORKED_OPTS \
1>"$TIKA_LOGS_DIR/tika-$TIKA_PORT-console.log" 2>&1 & echo $! > "$TIKA_PID_DIR/tika-$TIKA_PORT.pid"
# no lsof on cygwin though
@@ -547,7 +543,7 @@ function start_tika() {
exit # subshell!
fi
else
- TIKA_PID=`ps auxww | grep tika\-server | grep -w "\-p $TIKA_PORT" | grep -v "\-child" | grep -v grep | awk '{print $2}' | sort -r`
+ TIKA_PID=`ps auxww | grep tika\-server | grep -w "\-p $TIKA_PORT" | grep -v grep | awk '{print $2}' | sort -r`
echo -e "\nStarted Tika server on port $TIKA_PORT (pid=$TIKA_PID). Happy extracting!\n"
exit # subshell!
fi
@@ -556,7 +552,7 @@ function start_tika() {
else
echo -e "NOTE: Please install lsof as this script needs it to determine if Tika is listening on port $TIKA_PORT."
sleep 10
- TIKA_PID=`ps auxww | grep tika\-server | grep -w "\-p $TIKA_PORT" | grep -v "\-child" | grep -v grep | awk '{print $2}' | sort -r`
+ TIKA_PID=`ps auxww | grep tika\-server | grep -w "\-p $TIKA_PORT" | grep -v grep | awk '{print $2}' | sort -r`
echo -e "\nStarted Tika server on port $TIKA_PORT (pid=$TIKA_PID). Happy extracting!\n"
return;
fi
diff --git a/tika-server/tika-server-classic/bin/tika.in.sh b/tika-server/tika-server-classic/bin/tika.in.sh
index 687ceae..ac76ca6 100755
--- a/tika-server/tika-server-classic/bin/tika.in.sh
+++ b/tika-server/tika-server-classic/bin/tika.in.sh
@@ -73,7 +73,8 @@
# Sets the port Tika binds to, default is 9998
#TIKA_PORT=9998
-# Tika Server has a number of options to make it more robust to OOMs, Infinite
-# Loops, and Memory Leaks via the -spawnChild parameter. Learn more at
-# http://wiki.apache.org/tika/TikaJAXRS
-#TIKA_SPAWN_CHILD_OPTS=-spawnChild -maxFiles 100000 -JXmx4g
+# As of Tika 2.0, Tika Server forks a process so that the forked
+# process is more robust to OOMs, Infinite
+# Loops, and Memory Leaks. Learn more at
+# http://wiki.apache.org/tika/TikaJAXRS.
+#TIKA_FORKED_OPTS=-maxFiles 100000 -JXmx4g
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
index cec3024..5e12512 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
@@ -41,7 +41,7 @@ public class ServerStatusWatcher implements Runnable {
private final DataInputStream fromParent;
private final long maxFiles;
private final ServerTimeouts serverTimeouts;
- private final Path childStatusPath;
+ private final Path forkedStatusPath;
private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
@@ -49,13 +49,13 @@ public class ServerStatusWatcher implements Runnable {
private volatile Instant lastPing = null;
public ServerStatusWatcher(ServerStatus serverStatus,
- InputStream inputStream, Path childStatusPath,
+ InputStream inputStream, Path forkedStatusPath,
long maxFiles,
ServerTimeouts serverTimeouts) throws IOException {
this.serverStatus = serverStatus;
this.maxFiles = maxFiles;
this.serverTimeouts = serverTimeouts;
- this.childStatusPath = childStatusPath;
+ this.forkedStatusPath = forkedStatusPath;
serverStatus.setStatus(ServerStatus.STATUS.OPERATING);
this.fromParent = new DataInputStream(inputStream);
Thread statusWatcher = new Thread(new StatusWatcher());
@@ -109,7 +109,7 @@ public class ServerStatusWatcher implements Runnable {
private void writeStatus() throws IOException {
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
- try (FileChannel channel = FileChannel.open(childStatusPath,
+ try (FileChannel channel = FileChannel.open(forkedStatusPath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE)) {
while (elapsed < serverTimeouts.getPingTimeoutMillis()) {
@@ -170,21 +170,21 @@ public class ServerStatusWatcher implements Runnable {
}
//if something went wrong with the parent,
- //the child process should try to delete the tmp file
+ //the forked process should try to delete the tmp file
if (status == ServerStatus.STATUS.PARENT_EXCEPTION) {
try {
- Files.delete(childStatusPath);
+ Files.delete(forkedStatusPath);
} catch (IOException e) {
//swallow
}
}
- LOG.info("Shutting down child process with status: {}", status.name());
+ LOG.info("Shutting down forked process with status: {}", status.name());
System.exit(status.getShutdownCode());
}
//This is an internal thread that pulses every ServerTimeouts#pingPulseMillis
- //within the child to see if the child should die.
+ //within the forked process to see if the forked process should terminate.
private class StatusWatcher implements Runnable {
@Override
@@ -193,7 +193,7 @@ public class ServerStatusWatcher implements Runnable {
ServerStatus.STATUS currStatus = serverStatus.getStatus();
if (currStatus != ServerStatus.STATUS.OPERATING) {
- LOG.warn("child process observed "+currStatus.name()+ " and is shutting down.");
+ LOG.warn("forked process observed "+currStatus.name()+ " and is shutting down.");
shutdown(currStatus);
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeouts.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeouts.java
index 94bf704..5159c23 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeouts.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeouts.java
@@ -20,42 +20,42 @@ public class ServerTimeouts {
/*
TODO: integrate these settings:
- * Number of milliseconds to wait to start child process.
- public static final long DEFAULT_CHILD_PROCESS_STARTUP_MILLIS = 60000;
+ * Number of milliseconds to wait to start forked process.
+ public static final long DEFAULT_FORKED_PROCESS_STARTUP_MILLIS = 60000;
- * Maximum number of milliseconds to wait to shutdown child process to allow
+ * Maximum number of milliseconds to wait to shutdown forked process to allow
* for current parses to complete.
- public static final long DEFAULT_CHILD_PROCESS_SHUTDOWN_MILLIS = 30000;
+ public static final long DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS = 30000;
- private long childProcessStartupMillis = DEFAULT_CHILD_PROCESS_STARTUP_MILLIS;
+ private long forkedProcessStartupMillis = DEFAULT_FORKED_PROCESS_STARTUP_MILLIS;
- private long childProcessShutdownMillis = DEFAULT_CHILD_PROCESS_SHUTDOWN_MILLIS;
+ private long forkedProcessShutdownMillis = DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS;
*/
/**
- * If the child doesn't receive a ping or the parent doesn't
- * hear back from a ping in this amount of time, kill and restart the child.
+ * If the forked process doesn't receive a ping or the parent doesn't
+ * hear back from a ping in this amount of time, terminate and restart the forked process.
*/
public static final long DEFAULT_PING_TIMEOUT_MILLIS = 30000;
/**
- * How often should the parent try to ping the child to check status
+ * How often should the parent try to ping the forked process to check status
*/
public static final long DEFAULT_PING_PULSE_MILLIS = 500;
/**
* Number of milliseconds to wait per server task (parse, detect, unpack, translate,
- * etc.) before timing out and shutting down the child process.
+ * etc.) before timing out and shutting down the forked process.
*/
public static final long DEFAULT_TASK_TIMEOUT_MILLIS = 120000;
/**
- * Number of milliseconds to wait for child process to startup
+ * Number of milliseconds to wait for forked process to startup
*/
- public static final long DEFAULT_CHILD_STARTUP_MILLIS = 120000;
+ public static final long DEFAULT_FORKED_STARTUP_MILLIS = 120000;
private int maxRestarts = -1;
@@ -65,11 +65,11 @@ public class ServerTimeouts {
private long pingPulseMillis = DEFAULT_PING_PULSE_MILLIS;
- private long maxChildStartupMillis = DEFAULT_CHILD_STARTUP_MILLIS;
+ private long maxforkedStartupMillis = DEFAULT_FORKED_STARTUP_MILLIS;
/**
- * How long to wait for a task before shutting down the child server process
+ * How long to wait for a task before shutting down the forked server process
* and restarting it.
* @return
*/
@@ -94,8 +94,8 @@ public class ServerTimeouts {
*
* @param pingTimeoutMillis if the parent doesn't receive a response
* in this amount of time, or
- * if the child doesn't receive a ping
- * in this amount of time, restart the child process
+ * if the forked doesn't receive a ping
+ * in this amount of time, restart the forked process
*/
public void setPingTimeoutMillis(long pingTimeoutMillis) {
this.pingTimeoutMillis = pingTimeoutMillis;
@@ -107,7 +107,7 @@ public class ServerTimeouts {
/**
*
- * @param pingPulseMillis how often to test that the parent and/or child is alive
+ * @param pingPulseMillis how often to test that the parent and/or forked is alive
*/
public void setPingPulseMillis(long pingPulseMillis) {
this.pingPulseMillis = pingPulseMillis;
@@ -122,15 +122,15 @@ public class ServerTimeouts {
}
/**
- * Maximum time in millis to allow for the child process to startup
+ * Maximum time in millis to allow for the forked process to startup
* or restart
* @return
*/
- public long getMaxChildStartupMillis() {
- return maxChildStartupMillis;
+ public long getMaxForkedStartupMillis() {
+ return maxforkedStartupMillis;
}
- public void setMaxChildStartupMillis(long maxChildStartupMillis) {
- this.maxChildStartupMillis = maxChildStartupMillis;
+ public void setMaxForkedStartupMillis(long maxForkedStartupMillis) {
+ this.maxforkedStartupMillis = maxForkedStartupMillis;
}
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
index 805bf13..d4773ba 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
@@ -31,7 +31,7 @@ import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.cxf.binding.BindingFactoryManager;
@@ -75,7 +75,7 @@ import org.slf4j.LoggerFactory;
public class TikaServerCli {
- //used in spawn-child mode
+ //used in fork mode -- restart after processing this many files
private static final long DEFAULT_MAX_FILES = 100000;
@@ -92,11 +92,11 @@ public class TikaServerCli {
"drive or a webpage from your intranet. See CVE-2015-3271.\n"+
"Please make sure you know what you are doing.";
- private static final List<String> ONLY_IN_SPAWN_CHILD_MODE =
+ private static final List<String> ONLY_IN_FORK_MODE =
Arrays.asList(new String[] { "taskTimeoutMillis", "taskPulseMillis",
"pingTimeoutMillis", "pingPulseMillis", "maxFiles", "javaHome", "maxRestarts",
"numRestarts",
- "childStatusFile", "maxChildStartupMillis", "tmpFilePrefix"});
+ "forkedStatusFile", "maxForkedStartupMillis", "tmpFilePrefix"});
private static Options getOptions() {
Options options = new Options();
@@ -113,22 +113,32 @@ public class TikaServerCli {
options.addOption("?", "help", false, "this help message");
options.addOption("enableUnsecureFeatures", false, "this is required to enable fileUrl.");
options.addOption("enableFileUrl", false, "allows user to pass in fileUrl instead of InputStream.");
- options.addOption("spawnChild", false, "whether or not to spawn a child process for robustness");
- options.addOption("taskTimeoutMillis", true, "Only in spawn child mode: how long to wait for a task (e.g. parse) to finish");
- options.addOption("taskPulseMillis", true, "Only in spawn child mode: how often to check if a task has timed out.");
- options.addOption("pingTimeoutMillis", true, "Only in spawn child mode: how long to wait to wait for a ping and/or ping response.");
- options.addOption("pingPulseMillis", true, "Only in spawn child mode: how often to check if a ping has timed out.");
- options.addOption("maxChildStartupMillis", true, "Only in spawn child mode: Maximum number of millis to wait for the child process to startup.");
- options.addOption("maxRestarts", true, "Only in spawn child mode: how many times to restart child process, default is -1 (always restart)");
- options.addOption("maxFiles", true, "Only in spawn child mode: shutdown server after this many files (to handle parsers that might introduce " +
+ options.addOption("noFork", false, "legacy mode, less robust -- this starts up tika-server" +
+ " without forking a process.");
+ options.addOption("taskTimeoutMillis", true,
+ "Not allowed in -noFork: how long to wait for a task (e.g. parse) to finish");
+ options.addOption("taskPulseMillis", true,
+ "Not allowed in -noFork: how often to check if a task has timed out.");
+ options.addOption("pingTimeoutMillis", true,
+ "Not allowed in -noFork: how long to wait to wait for a ping and/or ping response.");
+ options.addOption("pingPulseMillis", true,
+ "Not allowed in -noFork: how often to check if a ping has timed out.");
+ options.addOption("maxForkedStartupMillis", true,
+ "Not allowed in -noFork: Maximum number of millis to wait for the forked process to startup.");
+ options.addOption("maxRestarts", true,
+ "Not allowed in -noFork: how many times to restart forked process, default is -1 (always restart)");
+ options.addOption("maxFiles", true,
+ "Not allowed in -noFork: shutdown server after this many files (to handle parsers that might introduce " +
"slowly building memory leaks); the default is "+DEFAULT_MAX_FILES +". Set to -1 to turn this off.");
- options.addOption("javaHome", true, "Only in spawn child mode: override system property JAVA_HOME for calling java for the child process");
- options.addOption("child", false, "Only in spawn child mode: this process is a child process -- do not use this! " +
- "Should only be invoked by parent process");
- options.addOption("childStatusFile", true, "Only in spawn child mode: temporary file used as mmap to communicate " +
- "with parent process -- do not use this! Should only be invoked by parent process.");
- options.addOption("tmpFilePrefix", true, "Only in spawn child mode: prefix for temp file - for debugging only");
- options.addOption("numRestarts", true, "Only in spawn child mode: number of times that the child has had to be restarted.");
+ options.addOption("javaHome", true,
+ "Not allowed in -noFork: override system property JAVA_HOME for calling java for the forked process");
+ options.addOption("forkedStatusFile", true,
+ "Not allowed in -noFork: temporary file used as to communicate " +
+ "with forking process -- do not use this! Should only be invoked by forking process.");
+ options.addOption("tmpFilePrefix", true,
+ "Not allowed in -noFork: prefix for temp file - for debugging only");
+ options.addOption("numRestarts", true,
+ "Not allowed in -noFork: number of times that the forked server has had to be restarted.");
return options;
}
@@ -146,31 +156,31 @@ public class TikaServerCli {
private static void execute(String[] args) throws Exception {
Options options = getOptions();
- CommandLineParser cliParser = new GnuParser();
+ CommandLineParser cliParser = new DefaultParser();
- //need to strip out -J (child jvm opts) from this parse
+ //need to strip out -J (forked jvm opts) from this parse
//they'll be processed correctly in args in the watch dog
//and they won't be needed in legacy.
- CommandLine line = cliParser.parse(options, stripChildArgs(args));
- if (line.hasOption("spawnChild")) {
- TikaServerWatchDog watchDog = new TikaServerWatchDog();
- watchDog.execute(args, configureServerTimeouts(line));
- } else {
- if (! line.hasOption("child")) {
+ CommandLine line = cliParser.parse(options, stripForkedArgs(args));
+ if (line.hasOption("noFork") || line.hasOption("forkedStatusFile")) {
+ if (line.hasOption("noFork")) {
//make sure the user didn't misunderstand the options
- for (String childOnly : ONLY_IN_SPAWN_CHILD_MODE) {
- if (line.hasOption(childOnly)) {
- System.err.println("The option '" + childOnly +
- "' can only be used with '-spawnChild'");
+ for (String forkedOnly : ONLY_IN_FORK_MODE) {
+ if (line.hasOption(forkedOnly)) {
+ System.err.println("The option '" + forkedOnly +
+ "' can't be used with '-noFork'");
usage(options);
}
}
}
- executeLegacy(line, options);
+ actuallyRunServer(line, options);
+ } else {
+ TikaServerWatchDog watchDog = new TikaServerWatchDog();
+ watchDog.execute(args, configureServerTimeouts(line));
}
}
- private static String[] stripChildArgs(String[] args) {
+ private static String[] stripForkedArgs(String[] args) {
List<String> ret = new ArrayList<>();
for (String arg : args) {
if (!arg.startsWith("-J")) {
@@ -180,7 +190,10 @@ public class TikaServerCli {
return ret.toArray(new String[0]);
}
- private static void executeLegacy(CommandLine line, Options options) throws Exception {
+ //This starts the server in this process. This can
+ //be either a direct call from -noFork or the process that is forked
+ //in the 2.0 default mode.
+ private static void actuallyRunServer(CommandLine line, Options options) throws Exception {
if (line.hasOption("help")) {
usage(options);
}
@@ -277,8 +290,10 @@ public class TikaServerCli {
String serverId = line.hasOption("i") ? line.getOptionValue("i") : UUID.randomUUID().toString();
LOG.debug("SERVER ID:" +serverId);
ServerStatus serverStatus;
- //if this is a child process
- if (line.hasOption("child")) {
+
+ if (line.hasOption("noFork")) {
+ serverStatus = new ServerStatus(serverId, 0, true);
+ } else {
serverStatus = new ServerStatus(serverId, Integer.parseInt(line.getOptionValue("numRestarts")),
false);
//redirect!!!
@@ -292,14 +307,12 @@ public class TikaServerCli {
}
ServerTimeouts serverTimeouts = configureServerTimeouts(line);
- String childStatusFile = line.getOptionValue("childStatusFile");
+ String forkedStatusFile = line.getOptionValue("forkedStatusFile");
Thread serverThread =
- new Thread(new ServerStatusWatcher(serverStatus, in,
- Paths.get(childStatusFile), maxFiles, serverTimeouts));
+ new Thread(new ServerStatusWatcher(serverStatus, in,
+ Paths.get(forkedStatusFile), maxFiles, serverTimeouts));
serverThread.start();
- } else {
- serverStatus = new ServerStatus(serverId, 0, true);
}
TikaResource.init(tika, digester, inputStreamFactory, serverStatus);
JAXRSServerFactoryBean sf = new JAXRSServerFactoryBean();
@@ -386,13 +399,13 @@ public class TikaServerCli {
private static ServerTimeouts configureServerTimeouts(CommandLine line) {
ServerTimeouts serverTimeouts = new ServerTimeouts();
/*TODO -- add these in
- if (line.hasOption("childProcessStartupMillis")) {
- serverTimeouts.setChildProcessStartupMillis(
- Long.parseLong(line.getOptionValue("childProcessStartupMillis")));
+ if (line.hasOption("forkedProcessStartupMillis")) {
+ serverTimeouts.setForkedProcessStartupMillis(
+ Long.parseLong(line.getOptionValue("forkedProcessStartupMillis")));
}
- if (line.hasOption("childProcessShutdownMillis")) {
- serverTimeouts.setChildProcessShutdownMillis(
- Long.parseLong(line.getOptionValue("childProcesShutdownMillis")));
+ if (line.hasOption("forkedProcessShutdownMillis")) {
+ serverTimeouts.setForkedProcessShutdownMillis(
+ Long.parseLong(line.getOptionValue("forkedProcesShutdownMillis")));
}*/
if (line.hasOption("taskTimeoutMillis")) {
serverTimeouts.setTaskTimeoutMillis(
@@ -411,9 +424,9 @@ public class TikaServerCli {
serverTimeouts.setMaxRestarts(Integer.parseInt(line.getOptionValue("maxRestarts")));
}
- if (line.hasOption("maxChildStartupMillis")) {
- serverTimeouts.setMaxChildStartupMillis(
- Long.parseLong(line.getOptionValue("maxChildStartupMillis")));
+ if (line.hasOption("maxForkedStartupMillis")) {
+ serverTimeouts.setMaxForkedStartupMillis(
+ Long.parseLong(line.getOptionValue("maxForkedStartupMillis")));
}
return serverTimeouts;
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
index 28973a7..6247463 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
@@ -48,19 +48,19 @@ import static java.nio.file.StandardOpenOption.WRITE;
public class TikaServerWatchDog {
- private enum CHILD_STATUS {
+ private enum FORKED_STATUS {
INITIALIZING,
RUNNING,
SHUTTING_DOWN
}
private static final Logger LOG = LoggerFactory.getLogger(TikaServerWatchDog.class);
- private static final String DEFAULT_CHILD_STATUS_FILE_PREFIX = "tika-server-child-process-mmap-";
+ private static final String DEFAULT_FORKED_STATUS_FILE_PREFIX = "tika-server-forked-process-mmap-";
- private Object[] childStatusLock = new Object[0];
- private volatile CHILD_STATUS childStatus = CHILD_STATUS.INITIALIZING;
+ private Object[] forkedStatusLock = new Object[0];
+ private volatile FORKED_STATUS forkedStatus = FORKED_STATUS.INITIALIZING;
private volatile Instant lastPing = null;
- private ChildProcess childProcess = null;
+ private ForkedProcess forkedProcess = null;
public void execute(String[] args, ServerTimeouts serverTimeouts) throws Exception {
@@ -69,19 +69,19 @@ public class TikaServerWatchDog {
startPingTimer(serverTimeouts);
try {
int restarts = 0;
- childProcess = new ChildProcess(args, restarts, serverTimeouts);
- setChildStatus(CHILD_STATUS.RUNNING);
+ forkedProcess = new ForkedProcess(args, restarts, serverTimeouts);
+ setForkedStatus(FORKED_STATUS.RUNNING);
while (true) {
- if (!childProcess.ping()) {
+ if (!forkedProcess.ping()) {
LOG.debug("bad ping, initializing");
restarts++;
- setChildStatus(CHILD_STATUS.INITIALIZING);
+ setForkedStatus(FORKED_STATUS.INITIALIZING);
lastPing = null;
- childProcess.close();
- LOG.debug("About to restart the child process");
- childProcess = new ChildProcess(args, restarts, serverTimeouts);
- LOG.info("Successfully restarted child process -- {} restarts so far)", restarts);
- setChildStatus(CHILD_STATUS.RUNNING);
+ forkedProcess.close();
+ LOG.debug("About to restart the forked process");
+ forkedProcess = new ForkedProcess(args, restarts, serverTimeouts);
+ LOG.info("Successfully restarted forked process -- {} restarts so far)", restarts);
+ setForkedStatus(FORKED_STATUS.RUNNING);
if (serverTimeouts.getMaxRestarts() > -1 && restarts >= serverTimeouts.getMaxRestarts()) {
LOG.warn("hit max restarts: "+restarts+". Stopping now");
break;
@@ -92,11 +92,11 @@ public class TikaServerWatchDog {
} catch (InterruptedException e) {
//interrupted...shutting down
} finally {
- setChildStatus(CHILD_STATUS.SHUTTING_DOWN);
+ setForkedStatus(FORKED_STATUS.SHUTTING_DOWN);
LOG.debug("about to shutdown");
- if (childProcess != null) {
+ if (forkedProcess != null) {
LOG.info("about to shutdown process");
- childProcess.close();
+ forkedProcess.close();
}
}
}
@@ -111,25 +111,25 @@ public class TikaServerWatchDog {
String[] newArgs = new String[args.length+2];
System.arraycopy(args, 0, newArgs, 0, args.length);
- newArgs[args.length] = "--id";
+ newArgs[args.length] = "-i";
newArgs[args.length+1] = UUID.randomUUID().toString();
return newArgs;
}
private void startPingTimer(ServerTimeouts serverTimeouts) {
- //if the child thread is in stop-the-world mode, and isn't
+ //if the forked thread is in stop-the-world mode, and isn't
//reading the ping, this thread checks to make sure
//that the parent ping is sent often enough.
//The write() in ping() could block.
//If there isn't a successful ping often enough,
- //this force destroys the child process.
+ //this force destroys the forked process.
Thread pingTimer = new Thread(new Runnable() {
@Override
public void run() {
while (true) {
long tmpLastPing = -1L;
- synchronized (childStatusLock) {
- if (childStatus == CHILD_STATUS.RUNNING) {
+ synchronized (forkedStatusLock) {
+ if (forkedStatus == FORKED_STATUS.RUNNING) {
tmpLastPing = lastPing.toEpochMilli();
}
}
@@ -138,11 +138,11 @@ public class TikaServerWatchDog {
if (elapsed > serverTimeouts.getPingTimeoutMillis()) {
Process processToDestroy = null;
try {
- processToDestroy = childProcess.process;
- LOG.warn("{} ms have elapsed since last successful ping. Destroying child now",
+ processToDestroy = forkedProcess.process;
+ LOG.warn("{} ms have elapsed since last successful ping. Destroying forked now",
elapsed);
- destroyChildForcibly(processToDestroy);
- childProcess.close();
+ destroyForkedForcibly(processToDestroy);
+ forkedProcess.close();
} catch (NullPointerException e) {
//ignore
}
@@ -162,16 +162,16 @@ public class TikaServerWatchDog {
}
- private void setChildStatus(CHILD_STATUS status) {
- synchronized (childStatusLock) {
- childStatus = status;
+ private void setForkedStatus(FORKED_STATUS status) {
+ synchronized (forkedStatusLock) {
+ forkedStatus = status;
}
}
private static List<String> extractArgs(String[] args) {
List<String> argList = new ArrayList<>();
for (int i = 0; i < args.length; i++) {
- if (args[i].startsWith("-J") || args[i].equals("-spawnChild") || args[i].equals("--spawnChild")) {
+ if (args[i].startsWith("-J")) {
continue;
}
if (args[i].equals("-javaHome")) {
@@ -227,56 +227,56 @@ public class TikaServerWatchDog {
return jvmArgs;
}
- private class ChildProcess {
+ private class ForkedProcess {
private Thread SHUTDOWN_HOOK = null;
private final Process process;
- private final DataOutputStream toChild;
+ private final DataOutputStream toForked;
private final ServerTimeouts serverTimeouts;
- private final Path childStatusFile;
+ private final Path forkedStatusFile;
private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
- private ChildProcess(String[] args, int numRestarts, ServerTimeouts serverTimeouts) throws Exception {
- String prefix = DEFAULT_CHILD_STATUS_FILE_PREFIX;
+ private ForkedProcess(String[] args, int numRestarts, ServerTimeouts serverTimeouts) throws Exception {
+ String prefix = DEFAULT_FORKED_STATUS_FILE_PREFIX;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-tmpFilePrefix")) {
prefix = args[i+1];
}
}
- this.childStatusFile = Files.createTempFile(prefix, "");
+ this.forkedStatusFile = Files.createTempFile(prefix, "");
this.serverTimeouts = serverTimeouts;
- this.process = startProcess(args, numRestarts, childStatusFile);
+ this.process = startProcess(args, numRestarts, forkedStatusFile);
- //wait for file to be written/initialized by child process
+ //wait for file to be written/initialized by forked process
Instant start = Instant.now();
long elapsed = Duration.between(start, Instant.now()).toMillis();
try {
- while (process.isAlive() && Files.size(childStatusFile) < 12
- && elapsed < serverTimeouts.getMaxChildStartupMillis()) {
+ while (process.isAlive() && Files.size(forkedStatusFile) < 12
+ && elapsed < serverTimeouts.getMaxForkedStartupMillis()) {
Thread.sleep(50);
elapsed = Duration.between(start, Instant.now()).toMillis();
}
} catch (IOException e) {
- //the childStatusFile can be deleted by the
- //child process if it closes...this can lead to a NoSuchFileException
- LOG.warn("failed to start child process", e);
+ //the forkedStatusFile can be deleted by the
+ //forked process if it closes...this can lead to a NoSuchFileException
+ LOG.warn("failed to start forked process", e);
}
- if (elapsed > serverTimeouts.getMaxChildStartupMillis()) {
+ if (elapsed > serverTimeouts.getMaxForkedStartupMillis()) {
close();
- throw new RuntimeException("Child process failed to start after "+elapsed + " (ms)");
+ throw new RuntimeException("Forked process failed to start after "+elapsed + " (ms)");
}
if (!process.isAlive()) {
close();
- throw new RuntimeException("Failed to start child process -- child is not alive");
+ throw new RuntimeException("Failed to start forked process -- forked is not alive");
}
- if (!Files.exists(childStatusFile)) {
+ if (!Files.exists(forkedStatusFile)) {
close();
- throw new RuntimeException("Failed to start child process -- child status file does not exist");
+ throw new RuntimeException("Failed to start forked process -- forked status file does not exist");
}
- this.toChild = new DataOutputStream(process.getOutputStream());
+ this.toForked = new DataOutputStream(process.getOutputStream());
lastPing = Instant.now();
}
@@ -286,36 +286,36 @@ public class TikaServerWatchDog {
return false;
}
try {
- toChild.writeByte(ServerStatus.DIRECTIVES.PING.getByte());
- toChild.flush();
+ toForked.writeByte(ServerStatus.DIRECTIVES.PING.getByte());
+ toForked.flush();
} catch (Exception e) {
- LOG.warn("Exception pinging child process", e);
+ LOG.warn("Exception pinging forked process", e);
return false;
}
- ChildStatus childStatus = null;
+ ForkedStatus forkedStatus = null;
try {
- childStatus = readStatus();
+ forkedStatus = readStatus();
} catch (Exception e) {
- LOG.warn("Exception reading status from child", e);
+ LOG.warn("Exception reading status from forked", e);
return false;
}
- if (childStatus.status != ServerStatus.STATUS.OPERATING.getInt()) {
- LOG.warn("Received non-operating status from child: {}",
- ServerStatus.STATUS.lookup(childStatus.status));
+ if (forkedStatus.status != ServerStatus.STATUS.OPERATING.getInt()) {
+ LOG.warn("Received non-operating status from forked: {}",
+ ServerStatus.STATUS.lookup(forkedStatus.status));
return false;
}
long elapsedSinceLastUpdate =
- Duration.between(Instant.ofEpochMilli(childStatus.timestamp), Instant.now()).toMillis();
- LOG.debug("last update: {}, elapsed:{}, status:{}", childStatus.timestamp, elapsedSinceLastUpdate,
- childStatus.status);
+ Duration.between(Instant.ofEpochMilli(forkedStatus.timestamp), Instant.now()).toMillis();
+ LOG.debug("last update: {}, elapsed:{}, status:{}", forkedStatus.timestamp, elapsedSinceLastUpdate,
+ forkedStatus.status);
if (elapsedSinceLastUpdate >
serverTimeouts.getPingTimeoutMillis()) {
- //child hasn't written a status update in a longer time than allowed
- LOG.warn("Child's last update exceeded ping timeout: {} (ms) with status {}",
- elapsedSinceLastUpdate, childStatus.status);
+ //forked hasn't written a status update in a longer time than allowed
+ LOG.warn("Forked's last update exceeded ping timeout: {} (ms) with status {}",
+ elapsedSinceLastUpdate, forkedStatus.status);
return false;
}
@@ -323,11 +323,11 @@ public class TikaServerWatchDog {
return true;
}
- private ChildStatus readStatus() throws Exception {
+ private ForkedStatus readStatus() throws Exception {
Instant started = Instant.now();
Long elapsed = Duration.between(started, Instant.now()).toMillis();
//only reading, but need to include write to allow for locking
- try (FileChannel fc = FileChannel.open(childStatusFile, READ, WRITE)) {
+ try (FileChannel fc = FileChannel.open(forkedStatusFile, READ, WRITE)) {
while (elapsed < serverTimeouts.getPingTimeoutMillis()) {
try (FileLock lock = fc.tryLock(0, 16, true)) {
@@ -337,7 +337,7 @@ public class TikaServerWatchDog {
long timestamp = statusBuffer.getLong(0);
int status = statusBuffer.getInt(8);
int numTasks = statusBuffer.getInt(12);
- return new ChildStatus(timestamp, status, numTasks);
+ return new ForkedStatus(timestamp, status, numTasks);
}
} catch (OverlappingFileLockException e) {
//swallow
@@ -352,37 +352,37 @@ public class TikaServerWatchDog {
private void close() {
try {
- if (toChild != null) {
- toChild.writeByte(ServerStatus.DIRECTIVES.SHUTDOWN.getByte());
- toChild.flush();
+ if (toForked != null) {
+ toForked.writeByte(ServerStatus.DIRECTIVES.SHUTDOWN.getByte());
+ toForked.flush();
}
} catch (IOException e) {
- LOG.debug("Exception asking child to shutdown", e);
+ LOG.debug("Exception asking forked process to shutdown", e);
}
try {
- if (toChild != null) {
- toChild.close();
+ if (toForked != null) {
+ toForked.close();
}
} catch (IOException e) {
- LOG.debug("Problem shutting down writer to child", e);
+ LOG.debug("Problem shutting down writer to forked process", e);
}
- destroyChildForcibly(process);
+ destroyForkedForcibly(process);
- if (childStatusFile != null) {
+ if (forkedStatusFile != null) {
try {
- if (Files.isRegularFile(childStatusFile)) {
- Files.delete(childStatusFile);
+ if (Files.isRegularFile(forkedStatusFile)) {
+ Files.delete(forkedStatusFile);
}
- LOG.debug("deleted "+childStatusFile);
+ LOG.debug("deleted "+forkedStatusFile);
} catch (IOException e) {
- LOG.warn("problem deleting child status file", e);
+ LOG.warn("problem deleting forked process status file", e);
}
}
}
- private Process startProcess(String[] args, int numRestarts, Path childStatusFile) throws IOException {
+ private Process startProcess(String[] args, int numRestarts, Path forkedStatusFile) throws IOException {
ProcessBuilder builder = new ProcessBuilder();
builder.redirectError(ProcessBuilder.Redirect.INHERIT);
@@ -390,10 +390,10 @@ public class TikaServerWatchDog {
List<String> argList = new ArrayList<>();
String javaPath = extractJavaPath(args);
List<String> jvmArgs = extractJVMArgs(args);
- List<String> childArgs = extractArgs(args);
+ List<String> forkedArgs = extractArgs(args);
- childArgs.add("-childStatusFile");
- childArgs.add(ProcessUtils.escapeCommandLine(childStatusFile.toAbsolutePath().toString()));
+ forkedArgs.add("-forkedStatusFile");
+ forkedArgs.add(ProcessUtils.escapeCommandLine(forkedStatusFile.toAbsolutePath().toString()));
argList.add(javaPath);
if (! jvmArgs.contains("-cp") && ! jvmArgs.contains("--classpath")) {
@@ -403,11 +403,10 @@ public class TikaServerWatchDog {
}
argList.addAll(jvmArgs);
argList.add("org.apache.tika.server.core.TikaServerCli");
- argList.addAll(childArgs);
- argList.add("-child");
- argList.add("--numRestarts");
+ argList.addAll(forkedArgs);
+ argList.add("-numRestarts");
argList.add(Integer.toString(numRestarts));
- LOG.info("child process commandline: " +argList.toString());
+ LOG.info("forked process commandline: " +argList.toString());
builder.command(argList);
Process process = builder.start();
//redirect stdout to parent stderr to avoid error msgs
@@ -443,12 +442,12 @@ public class TikaServerWatchDog {
gobbler.start();
}
- private static synchronized void destroyChildForcibly(Process process) {
+ private static synchronized void destroyForkedForcibly(Process process) {
process = process.destroyForcibly();
try {
boolean destroyed = process.waitFor(60, TimeUnit.SECONDS);
if (! destroyed) {
- LOG.error("Child process still alive after 60 seconds. " +
+ LOG.error("Forked process still alive after 60 seconds. " +
"Shutting down the parent.");
System.exit(1);
}
@@ -457,12 +456,12 @@ public class TikaServerWatchDog {
}
}
- private static class ChildStatus {
+ private static class ForkedStatus {
private final long timestamp;
private final int status;
private final int numTasks;
- public ChildStatus(long timestamp, int status, int numTasks) {
+ public ForkedStatus(long timestamp, int status, int numTasks) {
this.timestamp = timestamp;
this.status = status;
this.numTasks = numTasks;
@@ -470,7 +469,7 @@ public class TikaServerWatchDog {
@Override
public String toString() {
- return "ChildStatus{" +
+ return "ForkedStatus{" +
"timestamp=" + timestamp +
", status=" + status +
", numTasks=" + numTasks +
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
index 59592e3..74b4f24 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
@@ -90,7 +90,8 @@ public class TikaServerIntegrationTest extends TikaTest {
public static void staticSetup() throws Exception {
LogUtils.setLoggerClass(NullWebClientLogger.class);
LOG_FILE = Files.createTempFile("tika-server-integration", ".xml");
- Files.copy(TikaServerIntegrationTest.class.getResourceAsStream("/logging/log4j_child.xml"), LOG_FILE, StandardCopyOption.REPLACE_EXISTING);
+ Files.copy(TikaServerIntegrationTest.class.getResourceAsStream("/logging/log4j_forked.xml"),
+ LOG_FILE, StandardCopyOption.REPLACE_EXISTING);
}
@Before
@@ -132,7 +133,6 @@ public class TikaServerIntegrationTest extends TikaTest {
TikaServerCli.main(
new String[]{
"-maxFiles", "2000",
- "-spawnChild",
"-p", INTEGRATION_TEST_PORT,
"-tmpFilePrefix", "basic-"
});
@@ -154,7 +154,7 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild", "-JXmx256m",
+ "-JXmx256m",
"-p", INTEGRATION_TEST_PORT,
"-pingPulseMillis", "100",
"-tmpFilePrefix", "tika-server-oom"
@@ -176,7 +176,7 @@ public class TikaServerIntegrationTest extends TikaTest {
//oom may or may not cause an exception depending
//on the timing
}
- //give some time for the server to crash/kill itself
+ //give some time for the server to crash/terminate itself
Thread.sleep(2000);
try {
testBaseline();
@@ -193,10 +193,10 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild", "-JXmx256m",
+ "-JXmx256m",
"-p", INTEGRATION_TEST_PORT,
"-pingPulseMillis", "100",
- "--status",
+ "-status",
"-tmpFilePrefix", "tika-server-oom"
});
@@ -216,7 +216,7 @@ public class TikaServerIntegrationTest extends TikaTest {
//oom may or may not cause an exception depending
//on the timing
}
- //give some time for the server to crash/kill itself
+ //give some time for the server to crash/terminate itself
Thread.sleep(2000);
try {
testBaseline();
@@ -235,11 +235,11 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild", "-JXmx256m",
+ "-JXmx256m",
"-p", INTEGRATION_TEST_PORT,
"-pingPulseMillis", "100",
- "--status",
- "--id",
+ "-status",
+ "-id",
serverId,
"-tmpFilePrefix", "tika-server-oom"
@@ -259,7 +259,7 @@ public class TikaServerIntegrationTest extends TikaTest {
//oom may or may not cause an exception depending
//on the timing
}
- //give some time for the server to crash/kill itself
+ //give some time for the server to crash/terminate itself
Thread.sleep(2000);
try {
testBaseline();
@@ -299,7 +299,6 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild",
"-p", INTEGRATION_TEST_PORT,
"-tmpFilePrefix", "tika-server-systemexit"
@@ -318,7 +317,7 @@ public class TikaServerIntegrationTest extends TikaTest {
} catch (Exception e) {
//sys exit causes catchable problems for the client
}
- //give some time for the server to crash/kill itself
+ //give some time for the server to crash/terminate itself
Thread.sleep(2000);
try {
testBaseline();
@@ -335,7 +334,7 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild", "-p", INTEGRATION_TEST_PORT,
+ "-p", INTEGRATION_TEST_PORT,
"-taskTimeoutMillis", "10000", "-taskPulseMillis", "500",
"-pingPulseMillis", "500",
"-tmpFilePrefix", "tika-server-timeoutok"
@@ -370,7 +369,7 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild", "-p", INTEGRATION_TEST_PORT,
+ "-p", INTEGRATION_TEST_PORT,
"-taskTimeoutMillis", "10000", "-taskPulseMillis", "100",
"-pingPulseMillis", "100",
"-tmpFilePrefix", "tika-server-timeout"
@@ -405,7 +404,7 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild", "-JXms20m", "-JXmx10m",
+ "-JXms20m", "-JXmx10m",
"-p", INTEGRATION_TEST_PORT,
"-tmpFilePrefix", "tika-server-badargs"
@@ -432,7 +431,6 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild",
"-p", INTEGRATION_TEST_PORT,
"-taskTimeoutMillis", "60000", "-taskPulseMillis", "500",
"-pingPulseMillis", "100",
@@ -469,7 +467,6 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild",
"-p", INTEGRATION_TEST_PORT,
"-taskTimeoutMillis", "60000", "-taskPulseMillis", "500",
"-pingPulseMillis", "100"
@@ -504,7 +501,6 @@ public class TikaServerIntegrationTest extends TikaTest {
public void run() {
TikaServerCli.main(
new String[]{
- "-spawnChild",
"-p", INTEGRATION_TEST_PORT,
"-taskTimeoutMillis", "10000", "-taskPulseMillis", "500",
"-pingPulseMillis", "100", "-maxRestarts", "0",
@@ -571,7 +567,6 @@ public class TikaServerIntegrationTest extends TikaTest {
TikaServerCli.main(
new String[]{
"-maxFiles", "10",
- "-spawnChild",
"-taskTimeoutMillis", "10000", "-taskPulseMillis", "500",
"-p", INTEGRATION_TEST_PORT
});
diff --git a/tika-server/tika-server-core/src/test/resources/logging/log4j_child.xml b/tika-server/tika-server-core/src/test/resources/logging/log4j_forked.xml
similarity index 100%
rename from tika-server/tika-server-core/src/test/resources/logging/log4j_child.xml
rename to tika-server/tika-server-core/src/test/resources/logging/log4j_forked.xml