You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/02/11 15:47:30 UTC
[tika] branch TIKA-3293 created (now 918aff0)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch TIKA-3293
in repository https://gitbox.apache.org/repos/asf/tika.git.
at 918aff0 TIKA-3293 -- refactor to move most of the commandline options into a TikaConfig file
This branch includes the following new commits:
new 9d1ebbc TIKA-3293 -- WIP -- move to a config file for tika-server
new 918aff0 TIKA-3293 -- refactor to move most of the commandline options into a TikaConfig file
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[tika] 01/02: TIKA-3293 -- WIP -- move to a config file for
tika-server
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-3293
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 9d1ebbc7fba568749c8f3b0b4fd239f3621818f6
Author: tballison <ta...@apache.org>
AuthorDate: Wed Feb 10 15:44:04 2021 -0500
TIKA-3293 -- WIP -- move to a config file for tika-server
---
.../tika/server/core/ServerStatusWatcher.java | 21 +-
.../tika/server/core/ServerTimeoutConfig.java | 136 -----
.../org/apache/tika/server/core/TikaServerCli.java | 242 ++-------
.../apache/tika/server/core/TikaServerConfig.java | 552 +++++++++++++++++++++
.../apache/tika/server/core/TikaServerProcess.java | 317 ++++++------
.../tika/server/core/TikaServerWatchDog.java | 60 +--
.../core/writer/MetadataListMessageBodyWriter.java | 1 -
.../main/resources/tika-server-config-default.xml | 97 ++++
.../org/apache/tika/server/core/CXFTestBase.java | 2 +-
.../tika/server/core/TikaServerConfigTest.java | 20 +
.../server/core/TikaServerIntegrationTest.java | 28 +-
.../configs/tika-config-server-badjvmargs.xml | 31 ++
.../resources/configs/tika-config-server-basic.xml | 30 ++
.../configs/tika-config-server-timeout-10000.xml | 26 +
.../test/resources/configs/tika-config-server.xml | 29 ++
15 files changed, 1014 insertions(+), 578 deletions(-)
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
index 7ac0ffd..a7a5cd7 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
@@ -39,8 +39,7 @@ public class ServerStatusWatcher implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ServerStatusWatcher.class);
private final ServerStatus serverStatus;
private final DataInputStream fromParent;
- private final long maxFiles;
- private final ServerTimeoutConfig serverTimeouts;
+ private final TikaServerConfig tikaServerConfig;
private final Path forkedStatusPath;
private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
@@ -50,11 +49,9 @@ public class ServerStatusWatcher implements Runnable {
public ServerStatusWatcher(ServerStatus serverStatus,
InputStream inputStream, Path forkedStatusPath,
- long maxFiles,
- ServerTimeoutConfig serverTimeouts) throws IOException {
+ TikaServerConfig tikaServerConfig) throws IOException {
this.serverStatus = serverStatus;
- this.maxFiles = maxFiles;
- this.serverTimeouts = serverTimeouts;
+ this.tikaServerConfig = tikaServerConfig;
this.forkedStatusPath = forkedStatusPath;
serverStatus.setStatus(ServerStatus.STATUS.OPERATING);
this.fromParent = new DataInputStream(inputStream);
@@ -112,7 +109,7 @@ public class ServerStatusWatcher implements Runnable {
try (FileChannel channel = FileChannel.open(forkedStatusPath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE)) {
- while (elapsed < serverTimeouts.getPingTimeoutMillis()) {
+ while (elapsed < tikaServerConfig.getPingTimeoutMillis()) {
try (FileLock lock = channel.tryLock()) {
if (lock != null) {
((Buffer) statusBuffer).position(0);
@@ -131,11 +128,11 @@ public class ServerStatusWatcher implements Runnable {
}
private void checkForHitMaxFiles() {
- if (maxFiles < 0) {
+ if (tikaServerConfig.getMaxFiles() < 0) {
return;
}
long filesProcessed = serverStatus.getFilesProcessed();
- if (filesProcessed >= maxFiles) {
+ if (filesProcessed >= tikaServerConfig.getMaxFiles()) {
serverStatus.setStatus(ServerStatus.STATUS.HIT_MAX_FILES);
}
}
@@ -144,7 +141,7 @@ public class ServerStatusWatcher implements Runnable {
Instant now = Instant.now();
for (TaskStatus status : serverStatus.getTasks().values()) {
long millisElapsed = Duration.between(status.started, now).toMillis();
- if (millisElapsed > serverTimeouts.getTaskTimeoutMillis()) {
+ if (millisElapsed > tikaServerConfig.getTaskTimeoutMillis()) {
serverStatus.setStatus(ServerStatus.STATUS.TIMEOUT);
if (status.fileName.isPresent()) {
LOG.error("Timeout task {}, millis elapsed {}, file {}" +
@@ -199,13 +196,13 @@ public class ServerStatusWatcher implements Runnable {
if (lastPing != null) {
long elapsed = Duration.between(lastPing, Instant.now()).toMillis();
- if (elapsed > serverTimeouts.getPingTimeoutMillis()) {
+ if (elapsed > tikaServerConfig.getPingTimeoutMillis()) {
serverStatus.setStatus(ServerStatus.STATUS.PARENT_EXCEPTION);
shutdown(ServerStatus.STATUS.PARENT_EXCEPTION);
}
}
try {
- Thread.sleep(serverTimeouts.getPingPulseMillis());
+ Thread.sleep(tikaServerConfig.getPingPulseMillis());
} catch (InterruptedException e) {
return;
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeoutConfig.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeoutConfig.java
deleted file mode 100644
index 53bcd64..0000000
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerTimeoutConfig.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.server.core;
-
-public class ServerTimeoutConfig {
-
- /*
- TODO: integrate these settings:
- * Number of milliseconds to wait to start forked process.
- public static final long DEFAULT_FORKED_PROCESS_STARTUP_MILLIS = 60000;
-
- * Maximum number of milliseconds to wait to shutdown forked process to allow
- * for current parses to complete.
- public static final long DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS = 30000;
-
- private long forkedProcessStartupMillis = DEFAULT_FORKED_PROCESS_STARTUP_MILLIS;
-
- private long forkedProcessShutdownMillis = DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS;
-
- */
-
-
-
- /**
- * If the forked process doesn't receive a ping or the parent doesn't
- * hear back from a ping in this amount of time, terminate and restart the forked process.
- */
- public static final long DEFAULT_PING_TIMEOUT_MILLIS = 30000;
-
- /**
- * How often should the parent try to ping the forked process to check status
- */
- public static final long DEFAULT_PING_PULSE_MILLIS = 500;
-
- /**
- * Number of milliseconds to wait per server task (parse, detect, unpack, translate,
- * etc.) before timing out and shutting down the forked process.
- */
- public static final long DEFAULT_TASK_TIMEOUT_MILLIS = 120000;
-
- /**
- * Number of milliseconds to wait for forked process to startup
- */
- public static final long DEFAULT_FORKED_STARTUP_MILLIS = 120000;
-
- private int maxRestarts = -1;
-
- private long taskTimeoutMillis = DEFAULT_TASK_TIMEOUT_MILLIS;
-
- private long pingTimeoutMillis = DEFAULT_PING_TIMEOUT_MILLIS;
-
- private long pingPulseMillis = DEFAULT_PING_PULSE_MILLIS;
-
- private long maxforkedStartupMillis = DEFAULT_FORKED_STARTUP_MILLIS;
-
-
- /**
- * How long to wait for a task before shutting down the forked server process
- * and restarting it.
- * @return
- */
- public long getTaskTimeoutMillis() {
- return taskTimeoutMillis;
- }
-
- /**
- *
- * @param taskTimeoutMillis number of milliseconds to allow per task
- * (parse, detection, unzipping, etc.)
- */
- public void setTaskTimeoutMillis(long taskTimeoutMillis) {
- this.taskTimeoutMillis = taskTimeoutMillis;
- }
-
- public long getPingTimeoutMillis() {
- return pingTimeoutMillis;
- }
-
- /**
- *
- * @param pingTimeoutMillis if the parent doesn't receive a response
- * in this amount of time, or
- * if the forked doesn't receive a ping
- * in this amount of time, restart the forked process
- */
- public void setPingTimeoutMillis(long pingTimeoutMillis) {
- this.pingTimeoutMillis = pingTimeoutMillis;
- }
-
- public long getPingPulseMillis() {
- return pingPulseMillis;
- }
-
- /**
- *
- * @param pingPulseMillis how often to test that the parent and/or forked is alive
- */
- public void setPingPulseMillis(long pingPulseMillis) {
- this.pingPulseMillis = pingPulseMillis;
- }
-
- public int getMaxRestarts() {
- return maxRestarts;
- }
-
- public void setMaxRestarts(int maxRestarts) {
- this.maxRestarts = maxRestarts;
- }
-
- /**
- * Maximum time in millis to allow for the forked process to startup
- * or restart
- * @return
- */
- public long getMaxForkedStartupMillis() {
- return maxforkedStartupMillis;
- }
-
- public void setMaxForkedStartupMillis(long maxForkedStartupMillis) {
- this.maxforkedStartupMillis = maxForkedStartupMillis;
- }
-}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
index 14a2ff4..3e89f96 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
@@ -19,9 +19,7 @@ package org.apache.tika.server.core;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
@@ -40,78 +38,27 @@ import org.apache.tika.Tika;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class TikaServerCli {
-
-
- //used in fork mode -- restart after processing this many files
- private static final long DEFAULT_MAX_FILES = 100000;
+import static org.apache.tika.server.core.TikaServerConfig.DEFAULT_HOST;
+import static org.apache.tika.server.core.TikaServerConfig.DEFAULT_PORT;
+public class TikaServerCli {
- public static final int DEFAULT_PORT = 9998;
- private static final int DEFAULT_DIGEST_MARK_LIMIT = 20*1024*1024;
- public static final String DEFAULT_HOST = "localhost";
- public static final Set<String> LOG_LEVELS = new HashSet<>(Arrays.asList("debug", "info"));
private static final Logger LOG = LoggerFactory.getLogger(TikaServerCli.class);
- private static final String UNSECURE_WARNING =
- "WARNING: You have chosen to run tika-server with unsecure features enabled.\n"+
- "Whoever has access to your service now has the same read permissions\n"+
- "as you've given your fetchers and the same write permissions as your emitters.\n" +
- "Users could request and receive a sensitive file from your\n" +
- "drive or a webpage from your intranet and/or send malicious content to\n" +
- " your emitter endpoints. See CVE-2015-3271.\n"+
- "Please make sure you know what you are doing.";
-
- private static final List<String> ONLY_IN_FORK_MODE =
- Arrays.asList(new String[] { "taskTimeoutMillis", "taskPulseMillis",
- "pingTimeoutMillis", "pingPulseMillis", "maxFiles", "javaHome", "maxRestarts",
- "numRestarts",
- "forkedStatusFile", "maxForkedStartupMillis", "tmpFilePrefix"});
-
private static Options getOptions() {
Options options = new Options();
- options.addOption("C", "cors", true, "origin allowed to make CORS requests (default=NONE)\nall allowed if \"all\"");
- options.addOption("h", "host", true, "host name (default = " + DEFAULT_HOST + ", use * for all)");
+ options.addOption("h", "host", true, "host name (default = "
+ + DEFAULT_HOST + ", use * for all)");
options.addOption("p", "port", true,
- "listen port(s) (default = " + DEFAULT_PORT + ").\n" +
+ "listen port(s) (default = 9998)\n" +
"Can specify multiple ports with inclusive ranges (e.g. 9990-9999)\n" +
"or with comma delimited list (e.g. 9996,9998,9995)");
- options.addOption("c", "config", true, "Tika Configuration file to override default config with.");
- options.addOption("d", "digest", true, "include digest in metadata, e.g. md5,sha1:32,sha256");
- options.addOption("dml", "digestMarkLimit", true, "max number of bytes to mark on stream for digest");
- options.addOption("l", "log", true, "request URI log level ('debug' or 'info')");
- options.addOption("s", "includeStack", false, "whether or not to return a stack trace\nif there is an exception during 'parse'");
- options.addOption("i", "id", true, "id to use for server in server status endpoint");
- options.addOption("status", false, "enable the status endpoint");
options.addOption("?", "help", false, "this help message");
- options.addOption("enableUnsecureFeatures", false, "this is required to enable fetchers and emitters. "+
- " The user acknowledges that fetchers and emitters introduce potential security vulnerabilities.");
- options.addOption("noFork", false, "legacy mode, less robust -- this starts up tika-server" +
- " without forking a process.");
- options.addOption("taskTimeoutMillis", true,
- "Not allowed in -noFork: how long to wait for a task (e.g. parse) to finish");
- options.addOption("taskPulseMillis", true,
- "Not allowed in -noFork: how often to check if a task has timed out.");
- options.addOption("pingTimeoutMillis", true,
- "Not allowed in -noFork: how long to wait to wait for a ping and/or ping response.");
- options.addOption("pingPulseMillis", true,
- "Not allowed in -noFork: how often to check if a ping has timed out.");
- options.addOption("maxForkedStartupMillis", true,
- "Not allowed in -noFork: Maximum number of millis to wait for the forked process to startup.");
- options.addOption("maxRestarts", true,
- "Not allowed in -noFork: how many times to restart forked process, default is -1 (always restart)");
- options.addOption("maxFiles", true,
- "Not allowed in -noFork: shutdown server after this many files (to handle parsers that might introduce " +
- "slowly building memory leaks); the default is "+DEFAULT_MAX_FILES +". Set to -1 to turn this off.");
- options.addOption("javaHome", true,
- "Not allowed in -noFork: override system property JAVA_HOME for calling java for the forked process");
- options.addOption("forkedStatusFile", true,
- "Not allowed in -noFork: temporary file used as to communicate " +
- "with forking process -- do not use this! Should only be invoked by forking process.");
- options.addOption("tmpFilePrefix", true,
- "Not allowed in -noFork: prefix for temp file - for debugging only");
- options.addOption("numRestarts", true,
- "Not allowed in -noFork: number of times that the forked server has had to be restarted.");
+ options.addOption("c", "config", true, "tika-config file");
+
+ options.addOption("i", "id", true, "id to use for server in" +
+ " the server status endpoint and logging");
+
return options;
}
@@ -131,14 +78,13 @@ public class TikaServerCli {
CommandLineParser cliParser = new DefaultParser();
- CommandLine line = cliParser.parse(options, stripForkedArgs(args));
- String[] newArgs = addDefaults(line, args);
- line = cliParser.parse(options, stripForkedArgs(newArgs));
- if (line.hasOption("noFork")) {
- noFork(line, newArgs);
+ CommandLine line = cliParser.parse(options, args);
+ TikaServerConfig tikaServerConfig = TikaServerConfig.load(line);
+ if (tikaServerConfig.isNoFork()) {
+ noFork(tikaServerConfig);
} else {
try {
- mainLoop(line, newArgs);
+ mainLoop(tikaServerConfig);
} catch (InterruptedException e) {
e.printStackTrace();
//swallow
@@ -146,23 +92,15 @@ public class TikaServerCli {
}
}
- private static void mainLoop(CommandLine line, String[] origArgs) throws Exception {
-
- List<String> argList = new ArrayList<>();
- argList.addAll(Arrays.asList(origArgs));
-
- NonForkedValues nonForkedValues = extractNonForkedValues(argList);
- int maxRestarts = nonForkedValues.maxRestarts;
- List<PortIdPair> portIdPairs = getPortIdPairs(nonForkedValues.id, nonForkedValues.portString);
+ private static void mainLoop(TikaServerConfig tikaServerConfig) throws Exception {
- String[] args = argList.toArray(new String[0]);
+ List<PortIdPair> portIdPairs = getPortIdPairs(tikaServerConfig);
ExecutorService executorService = Executors.newFixedThreadPool(portIdPairs.size());
ExecutorCompletionService<WatchDogResult> executorCompletionService = new ExecutorCompletionService<>(executorService);
- ServerTimeoutConfig serverTimeoutConfig = configureServerTimeouts(line);
for (PortIdPair p : portIdPairs) {
executorCompletionService.submit(
- new TikaServerWatchDog(args, p.port, p.id,0, serverTimeoutConfig));
+ new TikaServerWatchDog(p.port, p.id,0, tikaServerConfig));
}
int finished = 0;
@@ -173,16 +111,17 @@ public class TikaServerCli {
LOG.debug("main loop future is available");
WatchDogResult result = future.get();
LOG.debug("main loop future: ({}); about to restart", result);
- if (maxRestarts < 0 || result.getNumRestarts() < maxRestarts) {
+ if (tikaServerConfig.getMaxRestarts() < 0 ||
+ result.getNumRestarts() < tikaServerConfig.getMaxRestarts()) {
System.err.println("starting up again");
executorCompletionService.submit(
- new TikaServerWatchDog(args, result.getPort(),
+ new TikaServerWatchDog(result.getPort(),
result.getId(),
- result.getNumRestarts(), serverTimeoutConfig));
+ result.getNumRestarts(), tikaServerConfig));
} else {
System.err.println("finished!");
LOG.warn("id {} with port {} has exceeded maxRestarts {}. Shutting down and not restarting.",
- result.getId(), result.getPort(), maxRestarts);
+ result.getId(), result.getPort(), tikaServerConfig.getMaxRestarts());
finished++;
}
}
@@ -203,145 +142,38 @@ public class TikaServerCli {
return ret.toArray(new String[0]);
}
- //removes and records values that either shouldn't go into the forked
- //process or need to be modified
- private static NonForkedValues extractNonForkedValues(List<String> args) {
- int idIndex = -1;
- int portIndex = -1;
- int maxRestartIndex = -1;
- NonForkedValues nonForked = new NonForkedValues();
-
- for (int i = 0; i < args.size()-1; i++) {
- if (args.get(i).equals("-i") || args.get(i).equals("--id")) {
- idIndex = i;
- nonForked.id = args.get(i+1);
- } else if (args.get(i).equals("-p") ||
- args.get(i).equals("--port") || args.get(i).equals("--ports")) {
- portIndex = i;
- nonForked.portString = args.get(i+1);
- } else if (args.get(i).equals("-maxRestarts")
- || args.get(i).equals("--maxRestarts")) {
- maxRestartIndex = i;
- nonForked.maxRestarts = Integer.parseInt(args.get(i+1));
- }
- }
-
-
- //now remove -i and -p and their values from args
- List<String> copy = new ArrayList<>();
- copy.addAll(args);
- args.clear();
- for(int i = 0; i < copy.size(); i++) {
- if (i == idIndex || i == portIndex || i == maxRestartIndex) {
- i++;
- continue;
- }
- args.add(copy.get(i));
- }
-
- return nonForked;
- }
-
- public static void noFork(CommandLine line, String[] args) {
- //make sure the user didn't misunderstand the options
- for (String forkedOnly : ONLY_IN_FORK_MODE) {
- if (line.hasOption(forkedOnly)) {
- System.err.println("The option '" + forkedOnly +
- "' can't be used with '-noFork'");
- usage(getOptions());
- }
- }
- if (line.hasOption("p")) {
- String val = line.getOptionValue("p");
- try {
- Integer.parseInt(val);
- } catch (NumberFormatException e) {
- System.err.println("-p must be a single integer in -noFork mode. I see: "+val);
- usage(getOptions());
- }
- }
- TikaServerProcess.main(args);
+ public static void noFork(TikaServerConfig tikaServerConfig) {
+ List<String> args = tikaServerConfig.getForkedProcessArgs(
+ tikaServerConfig.getPort(), tikaServerConfig.getIdBase());
+ TikaServerProcess.main(args.toArray(new String[0]));
}
- private static String[] addDefaults(CommandLine line, String[] args) {
- List<String> newArr = new ArrayList<>(Arrays.asList(args));
- if (! line.hasOption("p")) {
- newArr.add("-p");
- newArr.add(Integer.toString(DEFAULT_PORT));
- }
- if (! line.hasOption("h")) {
- newArr.add("-h");
- newArr.add(DEFAULT_HOST);
- }
-
- if (! line.hasOption("i")) {
- newArr.add("-i");
- newArr.add(UUID.randomUUID().toString());
- }
- return newArr.toArray(new String[0]);
+ private static void usage(Options options) {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.printHelp("tikaserver", options);
+ System.exit(-1);
}
- private static List<PortIdPair> getPortIdPairs(String idString, String portsArg) {
+ private static List<PortIdPair> getPortIdPairs(TikaServerConfig tikaServerConfig) {
List<PortIdPair> pairs = new ArrayList<>();
Matcher m = Pattern.compile("^(\\d+)-(\\d+)\\Z").matcher("");
- for (String val : portsArg.split(",")) {
+ for (String val : tikaServerConfig.getPortString().split(",")) {
m.reset(val);
if (m.find()) {
int min = Math.min(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)));
int max = Math.max(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)));
for (int i = min; i <= max; i++) {
- pairs.add(new PortIdPair(i, idString+"-"+i));
+ pairs.add(new PortIdPair(i, tikaServerConfig.getIdBase() + "-" + i));
}
} else {
- pairs.add(new PortIdPair(Integer.parseInt(val), idString+"-"+val));
+ pairs.add(new PortIdPair(Integer.parseInt(val),
+ tikaServerConfig.getIdBase() + "-"+val));
}
}
return pairs;
}
- private static void usage(Options options) {
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp("tikaserver", options);
- System.exit(-1);
- }
-
- private static ServerTimeoutConfig configureServerTimeouts(CommandLine line) {
- ServerTimeoutConfig serverTimeouts = new ServerTimeoutConfig();
- /*TODO -- add these in
- if (line.hasOption("forkedProcessStartupMillis")) {
- serverTimeouts.setForkedProcessStartupMillis(
- Long.parseLong(line.getOptionValue("forkedProcessStartupMillis")));
- }
- if (line.hasOption("forkedProcessShutdownMillis")) {
- serverTimeouts.setForkedProcessShutdownMillis(
- Long.parseLong(line.getOptionValue("forkedProcesShutdownMillis")));
- }*/
- if (line.hasOption("taskTimeoutMillis")) {
- serverTimeouts.setTaskTimeoutMillis(
- Long.parseLong(line.getOptionValue("taskTimeoutMillis")));
- }
- if (line.hasOption("pingTimeoutMillis")) {
- serverTimeouts.setPingTimeoutMillis(
- Long.parseLong(line.getOptionValue("pingTimeoutMillis")));
- }
- if (line.hasOption("pingPulseMillis")) {
- serverTimeouts.setPingPulseMillis(
- Long.parseLong(line.getOptionValue("pingPulseMillis")));
- }
-
- if (line.hasOption("maxRestarts")) {
- serverTimeouts.setMaxRestarts(Integer.parseInt(line.getOptionValue("maxRestarts")));
- }
-
- if (line.hasOption("maxForkedStartupMillis")) {
- serverTimeouts.setMaxForkedStartupMillis(
- Long.parseLong(line.getOptionValue("maxForkedStartupMillis")));
- }
-
- return serverTimeouts;
- }
-
private static class PortIdPair {
int port;
String id;
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
new file mode 100644
index 0000000..96fd0ef
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
@@ -0,0 +1,552 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.server.core;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.tika.Tika;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.utils.ProcessUtils;
+import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.XMLReaderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.UUID;
+
+public class TikaServerConfig {
+
+ //used in fork mode -- restart after processing this many files
+ private static final long DEFAULT_MAX_FILES = 100000;
+
+
+ public static final int DEFAULT_PORT = 9998;
+ private static final int DEFAULT_DIGEST_MARK_LIMIT = 20*1024*1024;
+ public static final String DEFAULT_HOST = "localhost";
+ public static final Set<String> LOG_LEVELS = new HashSet<>(Arrays.asList("debug", "info"));
+
+ private static final String UNSECURE_WARNING =
+ "WARNING: You have chosen to run tika-server with unsecure features enabled.\n"+
+ "Whoever has access to your service now has the same read permissions\n"+
+ "as you've given your fetchers and the same write permissions as your emitters.\n" +
+ "Users could request and receive a sensitive file from your\n" +
+ "drive or a webpage from your intranet and/or send malicious content to\n" +
+ " your emitter endpoints. See CVE-2015-3271.\n"+
+ "Please make sure you know what you are doing.";
+
+ private static final List<String> ONLY_IN_FORK_MODE =
+ Arrays.asList(new String[] { "taskTimeoutMillis", "taskPulseMillis",
+ "pingTimeoutMillis", "pingPulseMillis", "maxFiles", "javaHome", "maxRestarts",
+ "numRestarts",
+ "forkedStatusFile", "maxForkedStartupMillis", "tmpFilePrefix"});
+
+ /**
+ * Config with only the defaults
+ */
+ public static TikaServerConfig load() {
+ return new TikaServerConfig();
+ }
+
+ public static TikaServerConfig load(CommandLine commandLine) throws IOException, TikaException {
+
+ TikaServerConfig config = null;
+ if (commandLine.hasOption("c")) {
+ config = load(Paths.get(commandLine.getOptionValue("c")));
+ config.setConfigPath(commandLine.getOptionValue("c"));
+ } else {
+ config = new TikaServerConfig();
+ }
+
+ //overwrite with the commandline
+ if (commandLine.hasOption("p")) {
+ int port = -1;
+ try {
+ config.setPort(Integer.parseInt(commandLine.getOptionValue("p")));
+ config.setPortString(commandLine.getOptionValue("p"));
+ } catch (NumberFormatException e) {
+ config.setPortString(commandLine.getOptionValue("p"));
+ }
+ }
+ if (commandLine.hasOption("h")) {
+ config.setHost(commandLine.getOptionValue("h"));
+ }
+
+ if (commandLine.hasOption("i")) {
+ config.setId(commandLine.getOptionValue("i"));
+ }
+
+ if (commandLine.hasOption("numRestarts")) {
+ config.setNumRestarts(Integer.parseInt(commandLine.getOptionValue("numRestarts")));
+ }
+
+ if (commandLine.hasOption("forkedStatusFile")) {
+ config.setForkedStatusFile(commandLine.getOptionValue("forkedStatusFile"));
+ }
+ config.validateConsistency();
+ return config;
+ }
+
+ private void setPortString(String portString) {
+ this.portString = portString;
+ }
+
+ private void setId(String id) {
+ this.idBase = id;
+ }
+
+ public static TikaServerConfig load (Path p) throws IOException, TikaException {
+ try (InputStream is = Files.newInputStream(p)) {
+ return TikaServerConfig.load(is);
+ }
+ }
+
+ public static TikaServerConfig load(InputStream is) throws IOException, TikaException {
+ Node properties = null;
+ try {
+ properties = XMLReaderUtils.buildDOM(is).getDocumentElement();
+ } catch (SAXException e) {
+ throw new IOException(e);
+ }
+ if (! properties.getLocalName().equals("properties")) {
+ throw new TikaConfigException("expect settings as root node");
+ }
+ NodeList children = properties.getChildNodes();
+ TikaServerConfig config = new TikaServerConfig();
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.item(i);
+ if ("server".equals(child.getLocalName())) {
+ loadServerConfig(child, config);
+ }
+ }
+ config.validateConsistency();
+ return config;
+ }
+
+ private static void loadServerConfig(Node server, TikaServerConfig config)
+ throws TikaConfigException {
+ NodeList params = server.getChildNodes();
+ for (int i = 0; i < params.getLength(); i++) {
+ Node param = params.item(i);
+ String localName = param.getLocalName();
+ String txt = param.getTextContent();
+ if ("endpoints".equals(localName)) {
+ config.addEndPoints(loadStringList("endpoint", param.getChildNodes()));
+ } else if ("forkedJVMArgs".equals(localName)) {
+ config.addJVMArgs(loadStringList("arg", param.getChildNodes()));
+ } else if (localName != null && txt != null) {
+ if ("port".equals(localName)) {
+ config.setPortString(txt);
+ } else {
+ tryToSet(config, localName, txt);
+ }
+ }
+ }
+ }
+
+ private static void tryToSet(TikaServerConfig config, String localName, String txt) throws TikaConfigException {
+ String setter = "set"+localName.substring(0,1).toUpperCase(Locale.US)+localName.substring(1);
+ Class[] types = new Class[]{String.class, boolean.class, int.class, long.class};
+ for (Class t : types) {
+ try {
+ Method m = TikaServerConfig.class.getMethod(setter, t);
+ if (t == int.class) {
+ try {
+ m.invoke(config, Integer.parseInt(txt));
+ return;
+ } catch (IllegalAccessException|InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter "+setter, e);
+ }
+ } else if (t == long.class) {
+ try {
+ m.invoke(config, Long.parseLong(txt));
+ return;
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter " + setter, e);
+ }
+ } else if (t == boolean.class) {
+ try {
+ m.invoke(config, Boolean.parseBoolean(txt));
+ return;
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter " + setter, e);
+ }
+ } else {
+ try {
+ m.invoke(config, txt);
+ return;
+ } catch (IllegalAccessException|InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter "+setter, e);
+ }
+ }
+ } catch (NoSuchMethodException e) {
+ //swallow
+ }
+ }
+ throw new TikaConfigException("Couldn't find setter: "+setter);
+ }
+
+ private static List<String> loadStringList(String itemName, NodeList nodelist) {
+ List<String> list = new ArrayList<>();
+ for (int i = 0; i < nodelist.getLength(); i++) {
+ Node n = nodelist.item(i);
+ if (itemName.equals(n.getLocalName())) {
+ list.add(n.getTextContent());
+ }
+ }
+ return list;
+ }
+
+ /*
+ TODO: integrate these settings:
+ * Number of milliseconds to wait to start forked process.
+ public static final long DEFAULT_FORKED_PROCESS_STARTUP_MILLIS = 60000;
+
+ * Maximum number of milliseconds to wait to shutdown forked process to allow
+ * for current parses to complete.
+ public static final long DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS = 30000;
+
+ private long forkedProcessStartupMillis = DEFAULT_FORKED_PROCESS_STARTUP_MILLIS;
+
+ private long forkedProcessShutdownMillis = DEFAULT_FORKED_PROCESS_SHUTDOWN_MILLIS;
+
+ */
+
+
+
+ /**
+ * If the forked process doesn't receive a ping or the parent doesn't
+ * hear back from a ping in this amount of time, terminate and restart the forked process.
+ */
+ public static final long DEFAULT_PING_TIMEOUT_MILLIS = 30000;
+
+ /**
+ * How often should the parent try to ping the forked process to check status
+ */
+ public static final long DEFAULT_PING_PULSE_MILLIS = 500;
+
+ /**
+ * Number of milliseconds to wait per server task (parse, detect, unpack, translate,
+ * etc.) before timing out and shutting down the forked process.
+ */
+ public static final long DEFAULT_TASK_TIMEOUT_MILLIS = 120000;
+
+ /**
+ * Number of milliseconds to wait for forked process to startup
+ */
+ public static final long DEFAULT_FORKED_STARTUP_MILLIS = 120000;
+
+ private int maxRestarts = -1;
+ private long maxFiles = 100000;
+ private long taskTimeoutMillis = DEFAULT_TASK_TIMEOUT_MILLIS;
+ private long pingTimeoutMillis = DEFAULT_PING_TIMEOUT_MILLIS;
+ private long pingPulseMillis = DEFAULT_PING_PULSE_MILLIS;
+ private long maxforkedStartupMillis = DEFAULT_FORKED_STARTUP_MILLIS;
+ private boolean enableUnsecureFeatures = false;
+ private String cors = "";
+ private boolean returnStackTrace = false;
+ private boolean noFork = false;
+ private String tempFilePrefix = "tika-server-tmp-"; //can be set for debugging
+ private List<String> forkedJvmArgs = new ArrayList<>();
+ private String idBase = UUID.randomUUID().toString();
+ private String portString = Integer.toString(DEFAULT_PORT);
+ private int port = DEFAULT_PORT;
+ private String host = DEFAULT_HOST;
+
+ private int digestMarkLimit = DEFAULT_DIGEST_MARK_LIMIT;
+ private String digest = "";
+ //debug or info only
+ private String logLevel = "";
+ private Path configPath;
+ private List<String> endPoints = new ArrayList<>();
+
+ //these should only be set in the forked process
+ //and they are automatically set by the forking process
+ private String forkedStatusFile;
+ private int numRestarts = 0;
+
+ public boolean isNoFork() {
+ return noFork;
+ }
+
+ public String getPortString() {
+ return portString;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+ /**
+ * How long to wait for a task before shutting down the forked server process
+ * and restarting it.
+ * @return
+ */
+ public long getTaskTimeoutMillis() {
+ return taskTimeoutMillis;
+ }
+
+ /**
+ *
+ * @param taskTimeoutMillis number of milliseconds to allow per task
+ * (parse, detection, unzipping, etc.)
+ */
+ public void setTaskTimeoutMillis(long taskTimeoutMillis) {
+ this.taskTimeoutMillis = taskTimeoutMillis;
+ }
+
+ public long getPingTimeoutMillis() {
+ return pingTimeoutMillis;
+ }
+
+ /**
+ *
+ * @param pingTimeoutMillis if the parent doesn't receive a response
+ * in this amount of time, or
+ * if the forked doesn't receive a ping
+ * in this amount of time, restart the forked process
+ */
+ public void setPingTimeoutMillis(long pingTimeoutMillis) {
+ this.pingTimeoutMillis = pingTimeoutMillis;
+ }
+
+ public long getPingPulseMillis() {
+ return pingPulseMillis;
+ }
+
+ /**
+ *
+ * @param pingPulseMillis how often to test that the parent and/or forked is alive
+ */
+ public void setPingPulseMillis(long pingPulseMillis) {
+ this.pingPulseMillis = pingPulseMillis;
+ }
+
+ public int getMaxRestarts() {
+ return maxRestarts;
+ }
+
+ public void setMaxRestarts(int maxRestarts) {
+ this.maxRestarts = maxRestarts;
+ }
+
+ public void setHost(String host) {
+ if ("*".equals(host)) {
+ host = "0.0.0.0";
+ }
+ this.host = host;
+ }
+
+ /**
+ * Maximum time in millis to allow for the forked process to startup
+ * or restart
+ * @return
+ */
+ public long getMaxForkedStartupMillis() {
+ return maxforkedStartupMillis;
+ }
+
+ public void setMaxForkedStartupMillis(long maxForkedStartupMillis) {
+ this.maxforkedStartupMillis = maxForkedStartupMillis;
+ }
+
+ public List<String> getForkedProcessArgs(int port, String id) {
+ //these are the arguments for the forked process
+ List<String> args = new ArrayList<>();
+ args.add("-p");
+ args.add(Integer.toString(port));
+ args.add("-i");
+ args.add(id);
+ if (hasConfigFile()) {
+ args.add("-c");
+ args.add(
+ ProcessUtils.escapeCommandLine(
+ configPath.toAbsolutePath().toString()));
+ }
+ return args;
+ }
+
+ public String getIdBase() {
+ return idBase;
+ }
+
+ /**
+ * full path to the java executable
+ * @return
+ */
+ public String getJavaPath() {
+ return "java";
+ }
+
+ public List<String> getForkedJvmArgs() {
+ return forkedJvmArgs;
+ }
+
+ public String getTempFilePrefix() {
+ return tempFilePrefix;
+ }
+
+ public boolean isEnableUnsecureFeatures() {
+ return enableUnsecureFeatures;
+ }
+
+ private void validateConsistency() throws TikaConfigException {
+ if (host == null) {
+ throw new TikaConfigException("Must specify 'host'");
+ }
+ if (!StringUtils.isBlank(portString)) {
+ try {
+ setPort(Integer.parseInt(portString));
+ } catch (NumberFormatException e) {
+
+ }
+ }
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setLogLevel(String level) throws TikaConfigException {
+ if (level.equals("debug") || level.equals("info")) {
+ this.logLevel = level;
+ } else {
+ throw new TikaConfigException("log level must be one of: 'debug' or 'info'");
+ }
+ }
+ public String getLogLevel() {
+ return logLevel;
+ }
+
+ /**
+ *
+ * @return the origin url for cors, can be "*"
+ */
+ public String getCors() {
+ return cors;
+ }
+
+ public boolean hasConfigFile() {
+ return configPath != null;
+ }
+
+ public void setConfigPath(String path) {
+ this.configPath = Paths.get(path);
+ }
+
+ public Path getConfigPath() {
+ return configPath;
+ }
+
+ public int getDigestMarkLimit() {
+ return digestMarkLimit;
+ }
+
+ /**
+ * digest configuration string, e.g. md5 or sha256, alternately w 16 or 32 encoding,
+ * e.g. md5:32,sha256:16 would result in two digests per file
+ * @return
+ */
+ public String getDigest() {
+ return digest;
+ }
+
+
+ /**
+ * maximum number of files before the forked server restarts.
+ * This is useful for avoiding any slow-building memory leaks/bloat.
+ * @return
+ */
+ public long getMaxFiles() {
+ return maxFiles;
+ }
+
+ public void setMaxFiles(long maxFiles) {
+ this.maxFiles = maxFiles;
+ }
+
+ public boolean isReturnStackTrace() {
+ return returnStackTrace;
+ }
+
+ public void setReturnStackTrace(boolean returnStackTrace) {
+ this.returnStackTrace = returnStackTrace;
+ }
+
+ public List<String> getEndPoints() {
+ return endPoints;
+ }
+
+ public String getId() {
+ //TODO fix this
+ return idBase;
+ }
+
+ private void addEndPoints(List<String> endPoints) {
+ this.endPoints.addAll(endPoints);
+ }
+
+ private void addJVMArgs(List<String> args) {
+ this.forkedJvmArgs.addAll(args);
+ }
+
+ public void setEnableUnsecureFeatures(boolean enableUnsecureFeatures) {
+ this.enableUnsecureFeatures = enableUnsecureFeatures;
+ }
+
+ /******
+ * these should only be used in the commandline for a forked process
+ ******/
+
+
+ private void setNumRestarts(int numRestarts) {
+ this.numRestarts = numRestarts;
+ }
+
+ public int getNumRestarts() {
+ return numRestarts;
+ }
+
+ public String getForkedStatusFile() {
+ return forkedStatusFile;
+ }
+
+ private void setForkedStatusFile(String forkedStatusFile) {
+ this.forkedStatusFile = forkedStatusFile;
+ }
+
+}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
index 5135f55..69b561f 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
@@ -64,6 +64,7 @@ import org.apache.tika.server.core.writer.MetadataListMessageBodyWriter;
import org.apache.tika.server.core.writer.TarWriter;
import org.apache.tika.server.core.writer.TextMessageBodyWriter;
import org.apache.tika.server.core.writer.ZipWriter;
+import org.apache.tika.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,34 +101,11 @@ public class TikaServerProcess {
private static Options getOptions() {
Options options = new Options();
- options.addOption("C", "cors", true, "origin allowed to make CORS requests (default=NONE)\nall allowed if \"all\"");
options.addOption("h", "host", true, "host name, use * for all)");
options.addOption("p", "port", true, "listen port");
options.addOption("c", "config", true, "Tika Configuration file to override default config with.");
- options.addOption("d", "digest", true, "include digest in metadata, e.g. md5,sha1:32,sha256");
- options.addOption("dml", "digestMarkLimit", true, "max number of bytes to mark on stream for digest");
- options.addOption("l", "log", true, "request URI log level ('debug' or 'info')");
- options.addOption("s", "includeStack", false, "whether or not to return a stack trace\nif there is an exception during 'parse'");
options.addOption("i", "id", true, "id to use for server in server status endpoint");
- options.addOption("status", false, "enable the status endpoint");
options.addOption("?", "help", false, "this help message");
- options.addOption("enableUnsecureFeatures", false, "this is required to enable fetchers and emitters. " +
- " The user acknowledges that fetchers and emitters introduce potential security vulnerabilities.");
- options.addOption("noFork", false, "legacy mode, less robust -- this starts up tika-server" +
- " without forking a process.");
- options.addOption("taskTimeoutMillis", true,
- "Not allowed in -noFork: how long to wait for a task (e.g. parse) to finish");
- options.addOption("taskPulseMillis", true,
- "Not allowed in -noFork: how often to check if a task has timed out.");
- options.addOption("pingTimeoutMillis", true,
- "Not allowed in -noFork: how long to wait to wait for a ping and/or ping response.");
- options.addOption("pingPulseMillis", true,
- "Not allowed in -noFork: how often to check if a ping has timed out.");
- options.addOption("maxFiles", true,
- "Not allowed in -noFork: shutdown server after this many files (to handle parsers that might introduce " +
- "slowly building memory leaks); the default is " + DEFAULT_MAX_FILES + ". Set to -1 to turn this off.");
- options.addOption("javaHome", true,
- "Not allowed in -noFork: override system property JAVA_HOME for calling java for the forked process");
options.addOption("forkedStatusFile", true,
"Not allowed in -noFork: temporary file used to communicate " +
"with forking process -- do not use this! Should only be invoked by forking process.");
@@ -144,7 +122,9 @@ public class TikaServerProcess {
Options options = getOptions();
CommandLineParser cliParser = new DefaultParser();
CommandLine line = cliParser.parse(options, args);
- mainLoop(line, options);
+ TikaServerConfig tikaServerConfig = TikaServerConfig.load(line);
+
+ mainLoop(tikaServerConfig);
} catch (Exception e) {
e.printStackTrace();
LOG.error("Can't start: ", e);
@@ -152,18 +132,18 @@ public class TikaServerProcess {
}
}
- private static void mainLoop(CommandLine commandLine, Options options) throws Exception {
+ private static void mainLoop(TikaServerConfig tikaServerConfig) throws Exception {
AsyncResource asyncResource = null;
ArrayBlockingQueue<FetchEmitTuple> asyncFetchEmitQueue = null;
ArrayBlockingQueue<EmitData> asyncEmitData = null;
int numAsyncParserThreads = 10;
- if (commandLine.hasOption(ENABLE_UNSECURE_FEATURES)) {
+ if (tikaServerConfig.isEnableUnsecureFeatures()) {
asyncResource = new AsyncResource();
asyncFetchEmitQueue = asyncResource.getFetchEmitQueue(10000);
asyncEmitData = asyncResource.getEmitDataQueue(1000);
}
- ServerDetails serverDetails = initServer(commandLine, asyncResource);
+ ServerDetails serverDetails = initServer(tikaServerConfig, asyncResource);
ExecutorService executorService = Executors.newFixedThreadPool(numAsyncParserThreads+1);
ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(executorService);
@@ -188,79 +168,33 @@ public class TikaServerProcess {
}
//This returns the server, configured and ready to be started.
- private static ServerDetails initServer(CommandLine line,
+ private static ServerDetails initServer(TikaServerConfig tikaServerConfig,
AsyncResource asyncResource) throws Exception {
- String host = null;
-
- if (line.hasOption("host")) {
- host = line.getOptionValue("host");
- if ("*".equals(host)) {
- host = "0.0.0.0";
- }
- } else {
- throw new IllegalArgumentException("Must specify 'host'");
- }
-
- int port = -1;
-
- if (line.hasOption("port")) {
- port = Integer.valueOf(line.getOptionValue("port"));
- } else {
- throw new IllegalArgumentException("Must specify port");
- }
-
- boolean returnStackTrace = false;
- if (line.hasOption("includeStack")) {
- returnStackTrace = true;
- }
-
- TikaLoggingFilter logFilter = null;
- if (line.hasOption("log")) {
- String logLevel = line.getOptionValue("log");
- if (LOG_LEVELS.contains(logLevel)) {
- boolean isInfoLevel = "info".equals(logLevel);
- logFilter = new TikaLoggingFilter(isInfoLevel);
- } else {
- LOG.info("Unsupported request URI log level: {}", logLevel);
- }
- }
-
- CrossOriginResourceSharingFilter corsFilter = null;
- if (line.hasOption("cors")) {
- corsFilter = new CrossOriginResourceSharingFilter();
- String url = line.getOptionValue("cors");
- List<String> origins = new ArrayList<>();
- if (!url.equals("*")) origins.add(url); // Empty list allows all origins.
- corsFilter.setAllowOrigins(origins);
- }
+ String host = tikaServerConfig.getHost();
+ int port = tikaServerConfig.getPort();
// The Tika Configuration to use throughout
TikaConfig tika;
- if (line.hasOption("config")) {
- String configFilePath = line.getOptionValue("config");
- LOG.info("Using custom config: {}", configFilePath);
- tika = new TikaConfig(configFilePath);
+ if (tikaServerConfig.hasConfigFile()) {
+ LOG.info("Using custom config: {}",
+ tikaServerConfig.getConfigPath());
+ tika = new TikaConfig(tikaServerConfig.getConfigPath());
} else {
tika = TikaConfig.getDefaultConfig();
}
DigestingParser.Digester digester = null;
- if (line.hasOption("digest")) {
- int digestMarkLimit = DEFAULT_DIGEST_MARK_LIMIT;
- if (line.hasOption("dml")) {
- String dmlS = line.getOptionValue("dml");
- try {
- digestMarkLimit = Integer.parseInt(dmlS);
- } catch (NumberFormatException e) {
- throw new RuntimeException("Must have parseable int after digestMarkLimit(dml): " + dmlS);
- }
- }
+ if (! StringUtils.isBlank(tikaServerConfig.getDigest())) {
try {
- digester = new CommonsDigester(digestMarkLimit, line.getOptionValue("digest"));
+ digester = new CommonsDigester(
+ tikaServerConfig.getDigestMarkLimit(),
+ tikaServerConfig.getDigest());
} catch (IllegalArgumentException commonsException) {
try {
- digester = new BouncyCastleDigester(digestMarkLimit, line.getOptionValue("digest"));
+ digester = new BouncyCastleDigester(
+ tikaServerConfig.getDigestMarkLimit(),
+ tikaServerConfig.getDigest());
} catch (IllegalArgumentException bcException) {
throw new IllegalArgumentException("Tried both CommonsDigester (" + commonsException.getMessage() +
") and BouncyCastleDigester (" + bcException.getMessage() + ")", bcException);
@@ -269,83 +203,45 @@ public class TikaServerProcess {
}
InputStreamFactory inputStreamFactory = null;
- if (line.hasOption(ENABLE_UNSECURE_FEATURES)) {
+ if (tikaServerConfig.isEnableUnsecureFeatures()) {
inputStreamFactory = new FetcherStreamFactory(tika.getFetcherManager());
} else {
inputStreamFactory = new DefaultInputStreamFactory();
}
- logFetchersAndEmitters(line.hasOption(ENABLE_UNSECURE_FEATURES), tika);
- String serverId = line.hasOption("i") ? line.getOptionValue("i") : UUID.randomUUID().toString();
+ logFetchersAndEmitters(tikaServerConfig.isEnableUnsecureFeatures(), tika);
+
+ String serverId = tikaServerConfig.getId();
LOG.debug("SERVER ID:" + serverId);
ServerStatus serverStatus;
- if (line.hasOption("noFork")) {
+ if (tikaServerConfig.isNoFork()) {
serverStatus = new ServerStatus(serverId, 0, true);
} else {
- serverStatus = new ServerStatus(serverId, Integer.parseInt(line.getOptionValue("numRestarts")),
+ serverStatus = new ServerStatus(serverId,
+ tikaServerConfig.getNumRestarts(),
false);
//redirect!!!
InputStream in = System.in;
System.setIn(new ByteArrayInputStream(new byte[0]));
System.setOut(System.err);
- long maxFiles = DEFAULT_MAX_FILES;
- if (line.hasOption("maxFiles")) {
- maxFiles = Long.parseLong(line.getOptionValue("maxFiles"));
- }
-
- ServerTimeoutConfig serverTimeouts = configureServerTimeouts(line);
- String forkedStatusFile = line.getOptionValue("forkedStatusFile");
+ String forkedStatusFile = tikaServerConfig.getForkedStatusFile();
Thread serverThread =
new Thread(new ServerStatusWatcher(serverStatus, in,
- Paths.get(forkedStatusFile), maxFiles, serverTimeouts));
+ Paths.get(forkedStatusFile), tikaServerConfig));
serverThread.start();
}
TikaResource.init(tika, digester, inputStreamFactory, serverStatus);
JAXRSServerFactoryBean sf = new JAXRSServerFactoryBean();
- List<ResourceProvider> rCoreProviders = new ArrayList<>();
- rCoreProviders.add(new SingletonResourceProvider(new MetadataResource()));
- rCoreProviders.add(new SingletonResourceProvider(new RecursiveMetadataResource()));
- rCoreProviders.add(new SingletonResourceProvider(new DetectorResource(serverStatus)));
- rCoreProviders.add(new SingletonResourceProvider(new LanguageResource()));
- rCoreProviders.add(new SingletonResourceProvider(new TranslateResource(serverStatus)));
- rCoreProviders.add(new SingletonResourceProvider(new TikaResource()));
- rCoreProviders.add(new SingletonResourceProvider(new UnpackerResource()));
- rCoreProviders.add(new SingletonResourceProvider(new TikaMimeTypes()));
- rCoreProviders.add(new SingletonResourceProvider(new TikaDetectors()));
- rCoreProviders.add(new SingletonResourceProvider(new TikaParsers()));
- rCoreProviders.add(new SingletonResourceProvider(new TikaVersion()));
- if (line.hasOption(ENABLE_UNSECURE_FEATURES)) {
- rCoreProviders.add(new SingletonResourceProvider(new EmitterResource()));
- rCoreProviders.add(new SingletonResourceProvider(asyncResource));
- }
- rCoreProviders.addAll(loadResourceServices());
- if (line.hasOption("status")) {
- rCoreProviders.add(new SingletonResourceProvider(new TikaServerStatus(serverStatus)));
- }
- List<ResourceProvider> rAllProviders = new ArrayList<>(rCoreProviders);
- rAllProviders.add(new SingletonResourceProvider(new TikaWelcome(rCoreProviders)));
- sf.setResourceProviders(rAllProviders);
-
+ List<ResourceProvider> resourceProviders = new ArrayList<>();
List<Object> providers = new ArrayList<>();
- providers.add(new TarWriter());
- providers.add(new ZipWriter());
- providers.add(new CSVMessageBodyWriter());
- providers.add(new MetadataListMessageBodyWriter());
- providers.add(new JSONMessageBodyWriter());
- providers.add(new TextMessageBodyWriter());
- providers.addAll(loadWriterServices());
- providers.add(new TikaServerParseExceptionMapper(returnStackTrace));
- providers.add(new JSONObjWriter());
-
- if (logFilter != null) {
- providers.add(logFilter);
- }
- if (corsFilter != null) {
- providers.add(corsFilter);
- }
+ loadAllProviders(tikaServerConfig, asyncResource, serverStatus,
+ resourceProviders, providers);
+
+ sf.setResourceProviders(resourceProviders);
+
sf.setProviders(providers);
//set compression interceptors
@@ -368,6 +264,112 @@ public class TikaServerProcess {
return details;
}
+ private static void loadAllProviders(TikaServerConfig tikaServerConfig,
+ AsyncResource asyncResource, ServerStatus serverStatus,
+ List<ResourceProvider> resourceProviders,
+ List<Object> writers) {
+ List<ResourceProvider> tmpCoreProviders = loadCoreProviders(
+ tikaServerConfig, asyncResource, serverStatus);
+
+ resourceProviders.addAll(tmpCoreProviders);
+ resourceProviders.add(new SingletonResourceProvider(
+ new TikaWelcome(tmpCoreProviders)));
+
+ //for now, just load everything
+ writers.add(new TarWriter());
+ writers.add(new ZipWriter());
+ writers.add(new CSVMessageBodyWriter());
+ writers.add(new MetadataListMessageBodyWriter());
+ writers.add(new JSONMessageBodyWriter());
+ writers.add(new TextMessageBodyWriter());
+ writers.addAll(loadWriterServices());
+ writers.add(new TikaServerParseExceptionMapper(
+ tikaServerConfig.isReturnStackTrace()));
+ writers.add(new JSONObjWriter());
+
+ TikaLoggingFilter logFilter = null;
+ if (!StringUtils.isBlank(tikaServerConfig.getLogLevel())) {
+ String logLevel = tikaServerConfig.getLogLevel();
+ if (LOG_LEVELS.contains(logLevel)) {
+ boolean isInfoLevel = "info".equals(logLevel);
+ logFilter = new TikaLoggingFilter(isInfoLevel);
+ writers.add(logFilter);
+ } else {
+ LOG.warn("Unsupported request URI log level: {}", logLevel);
+ }
+ }
+
+ CrossOriginResourceSharingFilter corsFilter = null;
+ if (!StringUtils.isBlank(tikaServerConfig.getCors())) {
+ corsFilter = new CrossOriginResourceSharingFilter();
+ String url = tikaServerConfig.getCors();
+ List<String> origins = new ArrayList<>();
+ if (!url.equals("*")) origins.add(url); // Empty list allows all origins.
+ corsFilter.setAllowOrigins(origins);
+ writers.add(corsFilter);
+ }
+
+ }
+
+ private static List<ResourceProvider> loadCoreProviders(
+ TikaServerConfig tikaServerConfig, AsyncResource asyncResource, ServerStatus serverStatus) {
+ List<ResourceProvider> resourceProviders = new ArrayList<>();
+ if (tikaServerConfig.getEndPoints().size() == 0) {
+ resourceProviders.add(new SingletonResourceProvider(new MetadataResource()));
+ resourceProviders.add(new SingletonResourceProvider(new RecursiveMetadataResource()));
+ resourceProviders.add(new SingletonResourceProvider(new DetectorResource(serverStatus)));
+ resourceProviders.add(new SingletonResourceProvider(new LanguageResource()));
+ resourceProviders.add(new SingletonResourceProvider(new TranslateResource(serverStatus)));
+ resourceProviders.add(new SingletonResourceProvider(new TikaResource()));
+ resourceProviders.add(new SingletonResourceProvider(new UnpackerResource()));
+ resourceProviders.add(new SingletonResourceProvider(new TikaMimeTypes()));
+ resourceProviders.add(new SingletonResourceProvider(new TikaDetectors()));
+ resourceProviders.add(new SingletonResourceProvider(new TikaParsers()));
+ resourceProviders.add(new SingletonResourceProvider(new TikaVersion()));
+ if (tikaServerConfig.isEnableUnsecureFeatures()) {
+ resourceProviders.add(new SingletonResourceProvider(new EmitterResource()));
+ resourceProviders.add(new SingletonResourceProvider(asyncResource));
+ resourceProviders.add(new SingletonResourceProvider(new TikaServerStatus(serverStatus)));
+ }
+ resourceProviders.addAll(loadResourceServices());
+ return resourceProviders;
+ }
+ for (String endPoint : tikaServerConfig.getEndPoints()) {
+ if ("meta".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new MetadataResource()));
+ } else if ("rmeta".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new RecursiveMetadataResource()));
+ } else if ("detect".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new DetectorResource(serverStatus)));
+ } else if ("language".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new LanguageResource()));
+ } else if ("translate".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TranslateResource(serverStatus)));
+ } else if ("tika".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TikaResource()));
+ } else if ("unpack".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new UnpackerResource()));
+ } else if ("mime".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TikaMimeTypes()));
+ } else if ("detectors".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TikaDetectors()));
+ } else if ("parsers".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TikaParsers()));
+ } else if ("version".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TikaVersion()));
+ } else if ("emit".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new EmitterResource()));
+ } else if ("async".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(asyncResource));
+ } else if ("status".equals(endPoint)) {
+ resourceProviders.add(new SingletonResourceProvider(new TikaServerStatus(serverStatus)));
+ }
+ resourceProviders.addAll(loadResourceServices());
+ }
+ System.out.println("loaded "+resourceProviders);
+ return resourceProviders;
+ }
+
private static void logFetchersAndEmitters(boolean enableUnsecureFeatures, TikaConfig tika) {
if (enableUnsecureFeatures) {
StringBuilder sb = new StringBuilder();
@@ -431,41 +433,6 @@ public class TikaServerProcess {
System.exit(-1);
}
- private static ServerTimeoutConfig configureServerTimeouts(CommandLine line) {
- ServerTimeoutConfig serverTimeouts = new ServerTimeoutConfig();
- /*TODO -- add these in
- if (line.hasOption("forkedProcessStartupMillis")) {
- serverTimeouts.setForkedProcessStartupMillis(
- Long.parseLong(line.getOptionValue("forkedProcessStartupMillis")));
- }
- if (line.hasOption("forkedProcessShutdownMillis")) {
- serverTimeouts.setForkedProcessShutdownMillis(
- Long.parseLong(line.getOptionValue("forkedProcesShutdownMillis")));
- }*/
- if (line.hasOption("taskTimeoutMillis")) {
- serverTimeouts.setTaskTimeoutMillis(
- Long.parseLong(line.getOptionValue("taskTimeoutMillis")));
- }
- if (line.hasOption("pingTimeoutMillis")) {
- serverTimeouts.setPingTimeoutMillis(
- Long.parseLong(line.getOptionValue("pingTimeoutMillis")));
- }
- if (line.hasOption("pingPulseMillis")) {
- serverTimeouts.setPingPulseMillis(
- Long.parseLong(line.getOptionValue("pingPulseMillis")));
- }
-
- if (line.hasOption("maxRestarts")) {
- serverTimeouts.setMaxRestarts(Integer.parseInt(line.getOptionValue("maxRestarts")));
- }
-
- if (line.hasOption("maxForkedStartupMillis")) {
- serverTimeouts.setMaxForkedStartupMillis(
- Long.parseLong(line.getOptionValue("maxForkedStartupMillis")));
- }
-
- return serverTimeouts;
- }
private static class ServerDetails {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
index 3670553..5cfbf05 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
@@ -64,40 +64,29 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
private volatile Instant lastPing = null;
private ForkedProcess forkedProcess = null;
- private final String[] args;
private final int port;
private final String id;
private final int restarts;
- private final ServerTimeoutConfig serverTimeoutConfig;
+ private final TikaServerConfig tikaServerConfig;
- TikaServerWatchDog(String[] args, int port, String id, int restarts,
- ServerTimeoutConfig serverTimeoutConfig) {
- this.args = addPortAndId(args, port, id);
+ TikaServerWatchDog(int port, String id, int restarts,
+ TikaServerConfig tikaServerConfig) {
this.port = port;
this.id = id;
this.restarts = restarts;
- this.serverTimeoutConfig = serverTimeoutConfig;
+ this.tikaServerConfig = tikaServerConfig;
}
- private static String[] addPortAndId(String[] args, int port, String id) {
- List<String> newArgs = new ArrayList<>();
- newArgs.addAll(Arrays.asList(args));
- newArgs.add("-p");
- newArgs.add(Integer.toString(port));
- newArgs.add("-i");
- newArgs.add(id);
- return newArgs.toArray(new String[0]);
- }
@Override
public WatchDogResult call() throws Exception {
LOG.info("server watch dog is starting up");
try {
- forkedProcess = new ForkedProcess(args, restarts, serverTimeoutConfig);
+ forkedProcess = new ForkedProcess(restarts);
setForkedStatus(FORKED_STATUS.RUNNING);
- startPingTimer(serverTimeoutConfig);
+ startPingTimer();
while (forkedProcess.ping()) {
- Thread.sleep(serverTimeoutConfig.getPingPulseMillis());
+ Thread.sleep(tikaServerConfig.getPingPulseMillis());
}
} catch (InterruptedException e) {
//interrupted...shutting down
@@ -112,7 +101,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
return new WatchDogResult(port, id,restarts+1);
}
- private void startPingTimer(ServerTimeoutConfig serverTimeouts) {
+ private void startPingTimer() {
//if the forked thread is in stop-the-world mode, and isn't
//reading the ping, this thread checks to make sure
//that the parent ping is sent often enough.
@@ -131,7 +120,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
}
if (tmpLastPing > 0) {
long elapsed = Duration.between(Instant.ofEpochMilli(tmpLastPing), Instant.now()).toMillis();
- if (elapsed > serverTimeouts.getPingTimeoutMillis()) {
+ if (elapsed > tikaServerConfig.getPingTimeoutMillis()) {
Process processToDestroy = null;
try {
processToDestroy = forkedProcess.process;
@@ -145,7 +134,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
}
}
try {
- Thread.sleep(serverTimeouts.getPingPulseMillis());
+ Thread.sleep(tikaServerConfig.getPingPulseMillis());
} catch (InterruptedException e) {
//swallow
}
@@ -228,28 +217,21 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
private final Process process;
private final DataOutputStream toForked;
- private final ServerTimeoutConfig serverTimeoutConfig;
private final Path forkedStatusFile;
private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
- private ForkedProcess(String[] args, int numRestarts, ServerTimeoutConfig serverTimeoutConfig) throws Exception {
- String prefix = DEFAULT_FORKED_STATUS_FILE_PREFIX;
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-tmpFilePrefix")) {
- prefix = args[i+1];
- }
- }
+ private ForkedProcess(int numRestarts) throws Exception {
+ String prefix = tikaServerConfig.getTempFilePrefix();
this.forkedStatusFile = Files.createTempFile(prefix, "");
- this.serverTimeoutConfig = serverTimeoutConfig;
- this.process = startProcess(args, numRestarts, forkedStatusFile);
+ this.process = startProcess(numRestarts, forkedStatusFile);
//wait for file to be written/initialized by forked process
Instant start = Instant.now();
long elapsed = Duration.between(start, Instant.now()).toMillis();
try {
while (process.isAlive() && Files.size(forkedStatusFile) < 12
- && elapsed < serverTimeoutConfig.getMaxForkedStartupMillis()) {
+ && elapsed < tikaServerConfig.getMaxForkedStartupMillis()) {
Thread.sleep(50);
elapsed = Duration.between(start, Instant.now()).toMillis();
}
@@ -259,7 +241,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
LOG.warn("failed to start forked process", e);
}
- if (elapsed > serverTimeoutConfig.getMaxForkedStartupMillis()) {
+ if (elapsed > tikaServerConfig.getMaxForkedStartupMillis()) {
close();
throw new RuntimeException("Forked process failed to start after "+elapsed + " (ms)");
}
@@ -308,7 +290,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
forkedStatus.status);
if (elapsedSinceLastUpdate >
- serverTimeoutConfig.getPingTimeoutMillis()) {
+ tikaServerConfig.getPingTimeoutMillis()) {
//forked hasn't written a status update in a longer time than allowed
LOG.warn("Forked's last update exceeded ping timeout: {} (ms) with status {}",
elapsedSinceLastUpdate, forkedStatus.status);
@@ -325,7 +307,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
//only reading, but need to include write to allow for locking
try (FileChannel fc = FileChannel.open(forkedStatusFile, READ, WRITE)) {
- while (elapsed < serverTimeoutConfig.getPingTimeoutMillis()) {
+ while (elapsed < tikaServerConfig.getPingTimeoutMillis()) {
try (FileLock lock = fc.tryLock(0, 16, true)) {
if (lock != null) {
((Buffer)statusBuffer).position(0);
@@ -378,15 +360,15 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
}
- private Process startProcess(String[] args, int numRestarts, Path forkedStatusFile) throws IOException {
+ private Process startProcess(int numRestarts, Path forkedStatusFile) throws IOException {
ProcessBuilder builder = new ProcessBuilder();
builder.redirectError(ProcessBuilder.Redirect.INHERIT);
List<String> argList = new ArrayList<>();
- String javaPath = extractJavaPath(args);
- List<String> jvmArgs = extractJVMArgs(args);
- List<String> forkedArgs = extractArgs(args);
+ String javaPath = tikaServerConfig.getJavaPath();
+ List<String> jvmArgs = tikaServerConfig.getForkedJvmArgs();
+ List<String> forkedArgs = tikaServerConfig.getForkedProcessArgs(port, id);
forkedArgs.add("-forkedStatusFile");
forkedArgs.add(ProcessUtils.escapeCommandLine(forkedStatusFile.toAbsolutePath().toString()));
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
index 647da2d..434f8b3 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/MetadataListMessageBodyWriter.java
@@ -58,7 +58,6 @@ public class MetadataListMessageBodyWriter implements MessageBodyWriter<Metadata
WebApplicationException {
Writer writer = new OutputStreamWriter(entityStream, UTF_8);
JsonMetadataList.toJson(list.getMetadata(), writer);
- writer.flush();
entityStream.flush();
}
}
diff --git a/tika-server/tika-server-core/src/main/resources/tika-server-config-default.xml b/tika-server/tika-server-core/src/main/resources/tika-server-config-default.xml
new file mode 100644
index 0000000..95d8550
--- /dev/null
+++ b/tika-server/tika-server-core/src/main/resources/tika-server-config-default.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <server>
+ <!-- which port to start the server on. If you specify a range,
+ e.g. 9995-9998, TikaServerCli will start four forked servers,
+ one at each port. You can also specify multiple forked servers
+ via a comma-delimited value: 9995,9997.
+
+ -->
+ <port>9998</port>
+ <host>localhost</host>
+ <!-- if specified, this will be the id that is used in the
+ /status endpoint and elsewhere. If an id is specified
+ and more than one forked processes are invoked, each process
+ will have an id followed by the port, e.g my_id-9998. If a
+ forked server has to restart, it will maintain its original id.
+ If not specified, a UUID will be generated.
+ -->
+ <id></id>
+ <!-- whether or not to allow CORS requests. Set to 'all' if you
+ want to allow all CORS requests. Set to NONE or leave blank
+ if you do not want to enable CORS. -->
+ <cors>NONE</cors>
+ <!-- which digests to calculate, comma delimited (e.g. md5,sha256);
+ optionally specify encoding followed by a colon (e.g. "sha1:32").
+ Can be empty if you don't want to calculate a digest -->
+ <digest>sha256</digest>
+ <!-- how much to read to memory during the digest phase before
+ spooling to disc...only if digest is selected -->
+ <digestMarkLimit>1000000</digestMarkLimit>
+ <!-- request URI log level 'debug' or 'info' -->
+ <log>info</log>
+ <!-- whether or not to include the stacktrace when a parse exception happens
+ in the data returned to the user -->
+ <includeStack>false</includeStack>
+ <!-- whether or not to enable the status endpoint -->
+ <status>false</status>
+ <!-- If set to 'true', this runs tika server "in process"
+ in the legacy 1.x mode.
+ This means that the server will be susceptible to infinite loops
+ and crashes.
+ If set to 'false', the server will spawn a forked
+ process and restart the forked process on catastrophic failures
+ (this was called -spawnChild mode in 1.x).
+ nofork=false is the default in 2.x
+ -->
+ <nofork>false</nofork>
+ <!-- maximum time to allow per parse before shutting down and restarting
+ the forked parser. Not allowed if nofork=true. -->
+ <taskTimeoutMillis>300000</taskTimeoutMillis>
+ <!-- how often to check whether a parse has timed out.
+ Not allowed if nofork=true. -->
+ <taskPulseMillis>10000</taskPulseMillis>
+ <!-- maximum time to allow for a response from the forked process
+ before shutting it down and restarting it.
+ Not allowed if nofork=true. -->
+ <pingTimeoutMillis>60000</pingTimeoutMillis>
+ <!-- how often to check whether the fork process needs to be restarted
+ Not allowed if nofork=true. -->
+ <pingPulseMillis>10000</pingPulseMillis>
+ <!-- maximum amount of time to wait for a forked process to
+ start up.
+ Not allowed if nofork=true. -->
+ <maxForkedStartupMillis>120000</maxForkedStartupMillis>
+ <!-- maximum number of times to allow a specific forked process
+ to be restarted.
+ Not allowed if nofork=true. -->
+ <maxRestarts>-1</maxRestarts>
+ <!-- maximum files to parse per forked process before
+ restarting the forked process to clear potential
+ memory leaks.
+ Not allowed if nofork=true. -->
+ <maxFiles>100000</maxFiles>
+ <!-- if you want to specify a specific javaHome for
+ the forked process.
+ Not allowed if nofork=true. -->
+ <javaHome></javaHome>
+ <!-- this is for debugging only -->
+ <tmpFilePrefix></tmpFilePrefix>
+ </server>
+</properties>
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java
index 47ada93..23a5871 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java
@@ -61,7 +61,7 @@ public abstract class CXFTestBase {
private final static int DIGESTER_READ_LIMIT = 20*1024*1024;
protected static final String endPoint =
- "http://localhost:" + TikaServerCli.DEFAULT_PORT;
+ "http://localhost:" + TikaServerConfig.DEFAULT_PORT;
protected Server server;
private TikaConfig tika;
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java
new file mode 100644
index 0000000..ac9b6c3
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java
@@ -0,0 +1,20 @@
+package org.apache.tika.server.core;
+
+import org.apache.tika.config.TikaConfigTest;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TikaServerConfigTest {
+
+ @Test
+ public void testBasic() throws Exception {
+ TikaServerConfig config = TikaServerConfig.load(
+ TikaConfigTest.class.getResourceAsStream("/configs/tika-config-server.xml"));
+ assertEquals(-1, config.getMaxRestarts());
+ assertEquals(54321, config.getTaskTimeoutMillis());
+ assertEquals(true, config.isEnableUnsecureFeatures());
+ }
+}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
index 9e2d7d5..b9d58d7 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
@@ -19,8 +19,10 @@ package org.apache.tika.server.core;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.utils.ProcessUtils;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@@ -33,9 +35,11 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.ConnectException;
+import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.Permission;
import java.time.Duration;
@@ -47,6 +51,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
public class TikaServerIntegrationTest extends IntegrationTestBase {
@@ -61,9 +66,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
public void run() {
TikaServerCli.main(
new String[]{
- "-maxFiles", "100",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-"
+ "-c", getConfig("tika-config-server-basic.xml")
});
}
};
@@ -127,7 +130,6 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
"-pingPulseMillis", "100",
"-status",
"-tmpFilePrefix", "tika-server-oom"
-
});
}
};
@@ -333,10 +335,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
public void run() {
TikaServerCli.main(
new String[]{
- "-JXms20m", "-JXmx10m",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "tika-server-badargs"
-
+ "-c", getConfig("tika-config-server-badjvmargs.xml"),
});
}
};
@@ -352,6 +351,15 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
assertEquals(-1, i.get());
}
+ private String getConfig(String configName) {
+ try {
+ return ProcessUtils.escapeCommandLine(Paths.get(TikaServerIntegrationTest.class.
+ getResource("/configs/"+configName).toURI()).toAbsolutePath().toString());
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Test
public void testStdErrOutBasic() throws Exception {
final AtomicInteger i = new AtomicInteger();
@@ -466,7 +474,7 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
private void awaitServerStartup() throws Exception {
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
- WebClient client = WebClient.create(endPoint + "/tika").accept("text/plain");
+ WebClient client = WebClient.create(endPoint + "/").accept("text/html");
while (elapsed < MAX_WAIT_MS) {
try {
Response response = client.get();
@@ -576,6 +584,8 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
assertEquals(1, metadataList.size());
assertEquals("Nikolai Lobachevsky", metadataList.get(0).get("author"));
assertContains("hello world", metadataList.get(0).get("X-TIKA:content"));
+ return;
}
+ fail("should have completed within 3 tries");
}
}
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-badjvmargs.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-badjvmargs.xml
new file mode 100644
index 0000000..603ac8d
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-badjvmargs.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <server>
+ <port>9999</port>
+ <taskTimeoutMillis>54321</taskTimeoutMillis>
+ <enableUnsecureFeatures>true</enableUnsecureFeatures>
+ <forkedJVMArgs>
+ <arg>-Xms20m</arg>
+ <arg>-Xmx10m</arg>
+ </forkedJVMArgs>
+ <endpoints>
+ <endpoint>rmeta</endpoint>
+ </endpoints>
+ </server>
+</properties>
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
new file mode 100644
index 0000000..42a2c58
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <server>
+ <taskTimeoutMillis>120000</taskTimeoutMillis>
+ <port>9999</port>
+ <maxFiles>1000</maxFiles>
+ <forkedJVMArgs>
+ <arg>-Xmx512m</arg>
+ </forkedJVMArgs>
+ <endpoints>
+ <endpoint>rmeta</endpoint>
+ </endpoints>
+ </server>
+</properties>
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml
new file mode 100644
index 0000000..78ec922
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <server>
+ <taskTimeoutMillis>120000</taskTimeoutMillis>
+ <maxFiles>1000</maxFiles>
+ <forkedJVMArgs>
+ <arg>-Xmx512m</arg>
+ </forkedJVMArgs>
+ </server>
+</properties>
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server.xml
new file mode 100644
index 0000000..50a751e
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <server>
+ <taskTimeoutMillis>54321</taskTimeoutMillis>
+ <enableUnsecureFeatures>true</enableUnsecureFeatures>
+ <forkedJVMArgs>
+ <arg>-Xmx2g</arg>
+ </forkedJVMArgs>
+ <endpoints>
+ <endpoint>rmeta</endpoint>
+ </endpoints>
+ </server>
+</properties>
[tika] 02/02: TIKA-3293 -- refactor to move most of the commandline
options into a TikaConfig file
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-3293
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 918aff0fa449e68899228bac4540cd23486c4904
Author: tballison <ta...@apache.org>
AuthorDate: Thu Feb 11 10:47:14 2021 -0500
TIKA-3293 -- refactor to move most of the commandline options into a TikaConfig file
---
.../tika/server/core/ServerStatusWatcher.java | 25 +-
.../org/apache/tika/server/core/TikaServerCli.java | 64 ++--
.../apache/tika/server/core/TikaServerConfig.java | 327 ++++++++++------
.../apache/tika/server/core/TikaServerProcess.java | 3 +-
.../tika/server/core/TikaServerWatchDog.java | 111 ++----
.../tika/server/core/resource/EmitterResource.java | 4 +-
.../tika/server/core/IntegrationTestBase.java | 61 ++-
.../core/TikaServerAsyncIntegrationTest.java | 27 --
.../tika/server/core/TikaServerConfigTest.java | 10 +-
.../core/TikaServerEmitterIntegrationTest.java | 184 ++++-----
.../server/core/TikaServerIntegrationTest.java | 420 ++++++++-------------
.../resources/configs/tika-config-server-basic.xml | 1 +
...er-basic.xml => tika-config-server-emitter.xml} | 5 +-
.../configs/tika-config-server-timeout-10000.xml | 6 +-
.../src/test/resources/log4j.properties | 6 +-
15 files changed, 604 insertions(+), 650 deletions(-)
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
index a7a5cd7..aaff0a1 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/ServerStatusWatcher.java
@@ -42,7 +42,7 @@ public class ServerStatusWatcher implements Runnable {
private final TikaServerConfig tikaServerConfig;
private final Path forkedStatusPath;
private final ByteBuffer statusBuffer = ByteBuffer.allocate(16);
-
+ private volatile boolean shuttingDown = false;
private volatile Instant lastPing = null;
@@ -58,8 +58,7 @@ public class ServerStatusWatcher implements Runnable {
Thread statusWatcher = new Thread(new StatusWatcher());
statusWatcher.setDaemon(true);
statusWatcher.start();
- writeStatus();
-
+ writeStatus(false);
}
@Override
@@ -81,7 +80,7 @@ public class ServerStatusWatcher implements Runnable {
checkForTaskTimeouts();
}
try {
- writeStatus();
+ writeStatus(false);
} catch (Exception e) {
LOG.warn("Exception writing to parent", e);
serverStatus.setStatus(ServerStatus.STATUS.PARENT_EXCEPTION);
@@ -93,7 +92,7 @@ public class ServerStatusWatcher implements Runnable {
shutdown(ServerStatus.STATUS.PARENT_REQUESTED_SHUTDOWN);
} else if (directive == ServerStatus.DIRECTIVES.PING_ACTIVE_SERVER_TASKS.getByte()) {
try {
- writeStatus();
+ writeStatus(false);
} catch (Exception e) {
LOG.warn("Exception writing to parent", e);
serverStatus.setStatus(ServerStatus.STATUS.PARENT_EXCEPTION);
@@ -103,7 +102,14 @@ public class ServerStatusWatcher implements Runnable {
}
}
- private void writeStatus() throws IOException {
+ private synchronized void writeStatus(boolean shuttingDown) throws IOException {
+ if (this.shuttingDown == true) {
+ return;
+ }
+ if (shuttingDown == true) {
+ this.shuttingDown = true;
+ }
+
Instant started = Instant.now();
long elapsed = Duration.between(started, Instant.now()).toMillis();
try (FileChannel channel = FileChannel.open(forkedStatusPath,
@@ -147,21 +153,20 @@ public class ServerStatusWatcher implements Runnable {
LOG.error("Timeout task {}, millis elapsed {}, file {}" +
"consider increasing the allowable time with the " +
"-taskTimeoutMillis flag",
- status.task.toString(), Long.toString(millisElapsed), status.fileName.get());
+ status.task.toString(), millisElapsed, status.fileName.get());
} else {
LOG.error("Timeout task {}, millis elapsed {}; " +
"consider increasing the allowable time with the " +
"-taskTimeoutMillis flag",
- status.task.toString(), Long.toString(millisElapsed));
+ status.task.toString(), millisElapsed);
}
}
}
}
private void shutdown(ServerStatus.STATUS status) {
-
try {
- writeStatus();
+ writeStatus(true);
} catch (Exception e) {
LOG.debug("problem writing status before shutdown", e);
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
index 3e89f96..94488f1 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerCli.java
@@ -58,6 +58,8 @@ public class TikaServerCli {
options.addOption("i", "id", true, "id to use for server in" +
" the server status endpoint and logging");
+ options.addOption("noFork", false, "runs in legacy 1.x mode -- " +
+ "server runs in process and is not safely isolated in a forked process");
return options;
}
@@ -97,10 +99,13 @@ public class TikaServerCli {
List<PortIdPair> portIdPairs = getPortIdPairs(tikaServerConfig);
ExecutorService executorService = Executors.newFixedThreadPool(portIdPairs.size());
- ExecutorCompletionService<WatchDogResult> executorCompletionService = new ExecutorCompletionService<>(executorService);
+ ExecutorCompletionService<WatchDogResult> executorCompletionService =
+ new ExecutorCompletionService<>(executorService);
+ List<TikaServerWatchDog> watchers = new ArrayList<>();
for (PortIdPair p : portIdPairs) {
- executorCompletionService.submit(
- new TikaServerWatchDog(p.port, p.id,0, tikaServerConfig));
+ TikaServerWatchDog watcher = new TikaServerWatchDog(p.port, p.id, tikaServerConfig);
+ executorCompletionService.submit(watcher);
+ watchers.add(watcher);
}
int finished = 0;
@@ -108,26 +113,23 @@ public class TikaServerCli {
while (finished < portIdPairs.size()) {
Future<WatchDogResult> future = executorCompletionService.poll(1, TimeUnit.MINUTES);
if (future != null) {
+ System.err.println("main loop future");
LOG.debug("main loop future is available");
WatchDogResult result = future.get();
- LOG.debug("main loop future: ({}); about to restart", result);
- if (tikaServerConfig.getMaxRestarts() < 0 ||
- result.getNumRestarts() < tikaServerConfig.getMaxRestarts()) {
- System.err.println("starting up again");
- executorCompletionService.submit(
- new TikaServerWatchDog(result.getPort(),
- result.getId(),
- result.getNumRestarts(), tikaServerConfig));
- } else {
- System.err.println("finished!");
- LOG.warn("id {} with port {} has exceeded maxRestarts {}. Shutting down and not restarting.",
- result.getId(), result.getPort(), tikaServerConfig.getMaxRestarts());
+ System.err.println("main loop future get");
+ LOG.debug("main loop future: ({}); finished", result);
finished++;
- }
}
}
+ } catch (InterruptedException e) {
+ System.err.println("INTERRUPTED");
+ for (TikaServerWatchDog w : watchers) {
+ w.shutDown();
+ }
+ LOG.debug("thread interrupted", e);
} finally {
//this is just asking nicely...there is no guarantee!
+ System.err.println("shutting down");
executorService.shutdownNow();
}
}
@@ -156,20 +158,32 @@ public class TikaServerCli {
private static List<PortIdPair> getPortIdPairs(TikaServerConfig tikaServerConfig) {
List<PortIdPair> pairs = new ArrayList<>();
- Matcher m = Pattern.compile("^(\\d+)-(\\d+)\\Z").matcher("");
- for (String val : tikaServerConfig.getPortString().split(",")) {
- m.reset(val);
- if (m.find()) {
- int min = Math.min(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)));
- int max = Math.max(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)));
+ Matcher rangeMatcher = Pattern.compile("^(\\d+)-(\\d+)\\Z").matcher("");
+ String[] commaDelimited = tikaServerConfig.getPortString().split(",");
+ List<Integer> indivPorts = new ArrayList<>();
+ for (String val : commaDelimited) {
+ rangeMatcher.reset(val);
+ if (rangeMatcher.find()) {
+ int min = Math.min(Integer.parseInt(rangeMatcher.group(1)), Integer.parseInt(rangeMatcher.group(2)));
+ int max = Math.max(Integer.parseInt(rangeMatcher.group(1)), Integer.parseInt(rangeMatcher.group(2)));
for (int i = min; i <= max; i++) {
- pairs.add(new PortIdPair(i, tikaServerConfig.getIdBase() + "-" + i));
+ indivPorts.add(i);
}
} else {
- pairs.add(new PortIdPair(Integer.parseInt(val),
- tikaServerConfig.getIdBase() + "-"+val));
+ indivPorts.add(Integer.parseInt(val));
}
}
+ //if there's only one port, use only the idbase, otherwise append -$port
+ if (indivPorts.size() == 0) {
+ throw new IllegalArgumentException("Couldn't find any ports in: "+tikaServerConfig.getPortString());
+ } else if (indivPorts.size() == 1) {
+ pairs.add(new PortIdPair(indivPorts.get(0), tikaServerConfig.getIdBase()));
+ } else {
+ for (int p : indivPorts) {
+ pairs.add(new PortIdPair(p, tikaServerConfig.getIdBase() + "-" + p));
+ }
+ }
+
return pairs;
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
index 96fd0ef..332f428 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerConfig.java
@@ -17,15 +17,11 @@
package org.apache.tika.server.core;
import org.apache.commons.cli.CommandLine;
-import org.apache.tika.Tika;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.utils.ProcessUtils;
import org.apache.tika.utils.StringUtils;
import org.apache.tika.utils.XMLReaderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
@@ -39,7 +35,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@@ -82,8 +78,9 @@ public class TikaServerConfig {
public static TikaServerConfig load(CommandLine commandLine) throws IOException, TikaException {
TikaServerConfig config = null;
+ Set<String> settings = new HashSet<>();
if (commandLine.hasOption("c")) {
- config = load(Paths.get(commandLine.getOptionValue("c")));
+ config = load(Paths.get(commandLine.getOptionValue("c")), settings);
config.setConfigPath(commandLine.getOptionValue("c"));
} else {
config = new TikaServerConfig();
@@ -94,138 +91,43 @@ public class TikaServerConfig {
int port = -1;
try {
config.setPort(Integer.parseInt(commandLine.getOptionValue("p")));
- config.setPortString(commandLine.getOptionValue("p"));
} catch (NumberFormatException e) {
- config.setPortString(commandLine.getOptionValue("p"));
}
+ config.setPortString(commandLine.getOptionValue("p"));
+ settings.add("port");
}
if (commandLine.hasOption("h")) {
config.setHost(commandLine.getOptionValue("h"));
+ settings.add("host");
}
if (commandLine.hasOption("i")) {
config.setId(commandLine.getOptionValue("i"));
+ settings.add("id");
}
if (commandLine.hasOption("numRestarts")) {
config.setNumRestarts(Integer.parseInt(commandLine.getOptionValue("numRestarts")));
+ settings.add("numRestarts");
}
if (commandLine.hasOption("forkedStatusFile")) {
config.setForkedStatusFile(commandLine.getOptionValue("forkedStatusFile"));
+ settings.add("forkedStatusFile");
}
- config.validateConsistency();
- return config;
- }
-
- private void setPortString(String portString) {
- this.portString = portString;
- }
-
- private void setId(String id) {
- this.idBase = id;
- }
-
- public static TikaServerConfig load (Path p) throws IOException, TikaException {
- try (InputStream is = Files.newInputStream(p)) {
- return TikaServerConfig.load(is);
- }
- }
- public static TikaServerConfig load(InputStream is) throws IOException, TikaException {
- Node properties = null;
- try {
- properties = XMLReaderUtils.buildDOM(is).getDocumentElement();
- } catch (SAXException e) {
- throw new IOException(e);
- }
- if (! properties.getLocalName().equals("properties")) {
- throw new TikaConfigException("expect settings as root node");
- }
- NodeList children = properties.getChildNodes();
- TikaServerConfig config = new TikaServerConfig();
- for (int i = 0; i < children.getLength(); i++) {
- Node child = children.item(i);
- if ("server".equals(child.getLocalName())) {
- loadServerConfig(child, config);
- }
+ if (commandLine.hasOption("noFork")) {
+ config.setNoFork(true);
+ settings.add("noFork");
}
- config.validateConsistency();
+ config.validateConsistency(settings);
return config;
}
- private static void loadServerConfig(Node server, TikaServerConfig config)
- throws TikaConfigException {
- NodeList params = server.getChildNodes();
- for (int i = 0; i < params.getLength(); i++) {
- Node param = params.item(i);
- String localName = param.getLocalName();
- String txt = param.getTextContent();
- if ("endpoints".equals(localName)) {
- config.addEndPoints(loadStringList("endpoint", param.getChildNodes()));
- } else if ("forkedJVMArgs".equals(localName)) {
- config.addJVMArgs(loadStringList("arg", param.getChildNodes()));
- } else if (localName != null && txt != null) {
- if ("port".equals(localName)) {
- config.setPortString(txt);
- } else {
- tryToSet(config, localName, txt);
- }
- }
- }
- }
-
- private static void tryToSet(TikaServerConfig config, String localName, String txt) throws TikaConfigException {
- String setter = "set"+localName.substring(0,1).toUpperCase(Locale.US)+localName.substring(1);
- Class[] types = new Class[]{String.class, boolean.class, int.class, long.class};
- for (Class t : types) {
- try {
- Method m = TikaServerConfig.class.getMethod(setter, t);
- if (t == int.class) {
- try {
- m.invoke(config, Integer.parseInt(txt));
- return;
- } catch (IllegalAccessException|InvocationTargetException e) {
- throw new TikaConfigException("bad parameter "+setter, e);
- }
- } else if (t == long.class) {
- try {
- m.invoke(config, Long.parseLong(txt));
- return;
- } catch (IllegalAccessException | InvocationTargetException e) {
- throw new TikaConfigException("bad parameter " + setter, e);
- }
- } else if (t == boolean.class) {
- try {
- m.invoke(config, Boolean.parseBoolean(txt));
- return;
- } catch (IllegalAccessException | InvocationTargetException e) {
- throw new TikaConfigException("bad parameter " + setter, e);
- }
- } else {
- try {
- m.invoke(config, txt);
- return;
- } catch (IllegalAccessException|InvocationTargetException e) {
- throw new TikaConfigException("bad parameter "+setter, e);
- }
- }
- } catch (NoSuchMethodException e) {
- //swallow
- }
- }
- throw new TikaConfigException("Couldn't find setter: "+setter);
- }
-
- private static List<String> loadStringList(String itemName, NodeList nodelist) {
- List<String> list = new ArrayList<>();
- for (int i = 0; i < nodelist.getLength(); i++) {
- Node n = nodelist.item(i);
- if (itemName.equals(n.getLocalName())) {
- list.add(n.getTextContent());
- }
+ static TikaServerConfig load (Path p, Set<String> settings) throws IOException, TikaException {
+ try (InputStream is = Files.newInputStream(p)) {
+ return TikaServerConfig.load(is, settings);
}
- return list;
}
/*
@@ -254,7 +156,7 @@ public class TikaServerConfig {
/**
* How often should the parent try to ping the forked process to check status
*/
- public static final long DEFAULT_PING_PULSE_MILLIS = 500;
+ public static final long DEFAULT_PING_PULSE_MILLIS = 100;
/**
* Number of milliseconds to wait per server task (parse, detect, unpack, translate,
@@ -262,6 +164,8 @@ public class TikaServerConfig {
*/
public static final long DEFAULT_TASK_TIMEOUT_MILLIS = 120000;
+ public static final long DEFAULT_TASK_PULSE_MILLIS = 100;
+
/**
* Number of milliseconds to wait for forked process to startup
*/
@@ -270,6 +174,7 @@ public class TikaServerConfig {
private int maxRestarts = -1;
private long maxFiles = 100000;
private long taskTimeoutMillis = DEFAULT_TASK_TIMEOUT_MILLIS;
+ private long taskPulseMillis = DEFAULT_TASK_PULSE_MILLIS;
private long pingTimeoutMillis = DEFAULT_PING_TIMEOUT_MILLIS;
private long pingPulseMillis = DEFAULT_PING_PULSE_MILLIS;
private long maxforkedStartupMillis = DEFAULT_FORKED_STARTUP_MILLIS;
@@ -277,7 +182,7 @@ public class TikaServerConfig {
private String cors = "";
private boolean returnStackTrace = false;
private boolean noFork = false;
- private String tempFilePrefix = "tika-server-tmp-"; //can be set for debugging
+ private String tempFilePrefix = "apache-tika-server-forked-tmp-"; //can be set for debugging
private List<String> forkedJvmArgs = new ArrayList<>();
private String idBase = UUID.randomUUID().toString();
private String portString = Integer.toString(DEFAULT_PORT);
@@ -296,6 +201,18 @@ public class TikaServerConfig {
private String forkedStatusFile;
private int numRestarts = 0;
+ private void setNoFork(boolean noFork) {
+ this.noFork = noFork;
+ }
+
+ private void setPortString(String portString) {
+ this.portString = portString;
+ }
+
+ private void setId(String id) {
+ this.idBase = id;
+ }
+
public boolean isNoFork() {
return noFork;
}
@@ -424,7 +341,7 @@ public class TikaServerConfig {
return enableUnsecureFeatures;
}
- private void validateConsistency() throws TikaConfigException {
+ private void validateConsistency(Set<String> settings) throws TikaConfigException {
if (host == null) {
throw new TikaConfigException("Must specify 'host'");
}
@@ -435,6 +352,29 @@ public class TikaServerConfig {
}
}
+
+ if (isNoFork()) {
+ for (String onlyFork : ONLY_IN_FORK_MODE) {
+ if (settings.contains(onlyFork)) {
+ throw new TikaConfigException("Can't set param="
+ + onlyFork + "if you've selected noFork");
+ }
+ }
+ }
+ //add headless if not already configured
+ boolean foundHeadlessOption = false;
+ for (String arg : forkedJvmArgs) {
+ if (arg.contains("java.awt.headless")) {
+ foundHeadlessOption = true;
+ }
+ }
+ //if user has already specified headless...don't modify
+ if (! foundHeadlessOption) {
+ forkedJvmArgs.add("-Djava.awt.headless=true");
+ }
+
+
+
}
public String getHost() {
@@ -549,4 +489,157 @@ public class TikaServerConfig {
this.forkedStatusFile = forkedStatusFile;
}
+ public void setCors(String cors) {
+ this.cors = cors;
+ }
+
+ public void setTaskPulseMillis(long taskPulseMillis) {
+ this.taskPulseMillis = taskPulseMillis;
+ }
+
+ public void setMaxforkedStartupMillis(long maxforkedStartupMillis) {
+ this.maxforkedStartupMillis = maxforkedStartupMillis;
+ }
+
+ public void setDigestMarkLimit(int digestMarkLimit) {
+ this.digestMarkLimit = digestMarkLimit;
+ }
+
+ public void setDigest(String digest) {
+ this.digest = digest;
+ }
+
+ @Override
+ public String toString() {
+ return "TikaServerConfig{" +
+ "maxRestarts=" + maxRestarts +
+ ", maxFiles=" + maxFiles +
+ ", taskTimeoutMillis=" + taskTimeoutMillis +
+ ", pingTimeoutMillis=" + pingTimeoutMillis +
+ ", pingPulseMillis=" + pingPulseMillis +
+ ", maxforkedStartupMillis=" + maxforkedStartupMillis +
+ ", enableUnsecureFeatures=" + enableUnsecureFeatures +
+ ", cors='" + cors + '\'' +
+ ", returnStackTrace=" + returnStackTrace +
+ ", noFork=" + noFork +
+ ", tempFilePrefix='" + tempFilePrefix + '\'' +
+ ", forkedJvmArgs=" + forkedJvmArgs +
+ ", idBase='" + idBase + '\'' +
+ ", portString='" + portString + '\'' +
+ ", port=" + port +
+ ", host='" + host + '\'' +
+ ", digestMarkLimit=" + digestMarkLimit +
+ ", digest='" + digest + '\'' +
+ ", logLevel='" + logLevel + '\'' +
+ ", configPath=" + configPath +
+ ", endPoints=" + endPoints +
+ ", forkedStatusFile='" + forkedStatusFile + '\'' +
+ ", numRestarts=" + numRestarts +
+ '}';
+ }
+
+ static TikaServerConfig load(InputStream is, Set<String> settings) throws IOException, TikaException {
+ Node properties = null;
+ try {
+ properties = XMLReaderUtils.buildDOM(is).getDocumentElement();
+ } catch (SAXException e) {
+ throw new IOException(e);
+ }
+ if (! properties.getLocalName().equals("properties")) {
+ throw new TikaConfigException("expect settings as root node");
+ }
+ NodeList children = properties.getChildNodes();
+ TikaServerConfig config = new TikaServerConfig();
+
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.item(i);
+ if ("server".equals(child.getLocalName())) {
+ loadServerConfig(child, config, settings);
+ }
+ }
+ return config;
+ }
+
+ private static Set<String> loadServerConfig(Node server,
+ TikaServerConfig config, Set<String> settings)
+ throws TikaConfigException {
+ NodeList params = server.getChildNodes();
+ for (int i = 0; i < params.getLength(); i++) {
+ Node param = params.item(i);
+ String localName = param.getLocalName();
+ String txt = param.getTextContent();
+ if ("configPath".equals(localName)) {
+ throw new TikaConfigException("can't currently " +
+ "set config path within a config file");
+ }
+ if ("endpoints".equals(localName)) {
+ config.addEndPoints(loadStringList("endpoint", param.getChildNodes()));
+ } else if ("forkedJVMArgs".equals(localName)) {
+ config.addJVMArgs(loadStringList("arg", param.getChildNodes()));
+ } else if (localName != null && txt != null) {
+ if ("port".equals(localName)) {
+ config.setPortString(txt);
+ } else {
+ tryToSet(config, localName, txt);
+ }
+ }
+ if (localName != null && txt != null) {
+ settings.add(localName);
+ }
+ }
+ return settings;
+ }
+
+ private static void tryToSet(TikaServerConfig config, String name, String value) throws TikaConfigException {
+ String setter = "set"+name.substring(0,1).toUpperCase(Locale.US)+name.substring(1);
+ Class[] types = new Class[]{String.class, boolean.class, int.class, long.class};
+ for (Class t : types) {
+ try {
+ Method m = TikaServerConfig.class.getMethod(setter, t);
+ if (t == int.class) {
+ try {
+ m.invoke(config, Integer.parseInt(value));
+ return;
+ } catch (IllegalAccessException|InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter "+setter, e);
+ }
+ } else if (t == long.class) {
+ try {
+ m.invoke(config, Long.parseLong(value));
+ return;
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter " + setter, e);
+ }
+ } else if (t == boolean.class) {
+ try {
+ m.invoke(config, Boolean.parseBoolean(value));
+ return;
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter " + setter, e);
+ }
+ } else {
+ try {
+ m.invoke(config, value);
+ return;
+ } catch (IllegalAccessException|InvocationTargetException e) {
+ throw new TikaConfigException("bad parameter "+setter, e);
+ }
+ }
+ } catch (NoSuchMethodException e) {
+ //swallow
+ }
+ }
+ throw new TikaConfigException("Couldn't find setter: "+setter);
+ }
+
+ private static List<String> loadStringList(String itemName, NodeList nodelist) {
+ List<String> list = new ArrayList<>();
+ for (int i = 0; i < nodelist.getLength(); i++) {
+ Node n = nodelist.item(i);
+ if (itemName.equals(n.getLocalName())) {
+ list.add(n.getTextContent());
+ }
+ }
+ return list;
+ }
}
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
index 69b561f..f1d9813 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java
@@ -123,7 +123,7 @@ public class TikaServerProcess {
CommandLineParser cliParser = new DefaultParser();
CommandLine line = cliParser.parse(options, args);
TikaServerConfig tikaServerConfig = TikaServerConfig.load(line);
-
+ LOG.debug("forked config: {}", tikaServerConfig);
mainLoop(tikaServerConfig);
} catch (Exception e) {
e.printStackTrace();
@@ -153,6 +153,7 @@ public class TikaServerProcess {
executorCompletionService.submit(new AsyncParser(asyncFetchEmitQueue, asyncEmitData));
}
}
+ System.out.println("starting server again? " + tikaServerConfig);
//start the server
Server server = serverDetails.sf.create();
LOG.info("Started Apache Tika server {} at {}",
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
index 5cfbf05..ddddd88 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerWatchDog.java
@@ -66,39 +66,50 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
private final int port;
private final String id;
- private final int restarts;
+ private int restarts = 0;
private final TikaServerConfig tikaServerConfig;
+ private volatile boolean shutDown = false;
- TikaServerWatchDog(int port, String id, int restarts,
+ TikaServerWatchDog(int port, String id,
TikaServerConfig tikaServerConfig) {
this.port = port;
this.id = id;
- this.restarts = restarts;
this.tikaServerConfig = tikaServerConfig;
}
@Override
public WatchDogResult call() throws Exception {
- LOG.info("server watch dog is starting up");
- try {
- forkedProcess = new ForkedProcess(restarts);
- setForkedStatus(FORKED_STATUS.RUNNING);
- startPingTimer();
- while (forkedProcess.ping()) {
- Thread.sleep(tikaServerConfig.getPingPulseMillis());
+ while (true) {
+ if (tikaServerConfig.getMaxRestarts() > 0 && restarts >=
+ tikaServerConfig.getMaxRestarts()) {
+ LOG.warn("hit max restarts ({}). Ending processing for {} {}",
+ restarts, id, port);
+ return new WatchDogResult(port, id, restarts);
}
- } catch (InterruptedException e) {
- //interrupted...shutting down
- } finally {
- setForkedStatus(FORKED_STATUS.SHUTTING_DOWN);
- LOG.debug("about to shutdown");
- if (forkedProcess != null) {
- LOG.info("about to shutdown process");
- forkedProcess.close();
+
+ try {
+ forkedProcess = new ForkedProcess(restarts++);
+ setForkedStatus(FORKED_STATUS.RUNNING);
+ startPingTimer();
+ while (forkedProcess.ping()) {
+ Thread.sleep(tikaServerConfig.getPingPulseMillis());
+ }
+ } catch (InterruptedException e) {
+ return new WatchDogResult(port, id,restarts);
+ } finally {
+ setForkedStatus(FORKED_STATUS.SHUTTING_DOWN);
+ LOG.debug("about to shutdown");
+ if (forkedProcess != null) {
+ LOG.info("about to shutdown process");
+ forkedProcess.close();
+ }
}
}
- return new WatchDogResult(port, id,restarts+1);
+ }
+
+ public void shutDown() {
+ shutDown = true;
}
private void startPingTimer() {
@@ -153,65 +164,6 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
}
}
- private static List<String> extractArgs(String[] args) {
- List<String> argList = new ArrayList<>();
- for (int i = 0; i < args.length; i++) {
- if (args[i].startsWith("-J")) {
- continue;
- }
- if (args[i].equals("-javaHome")) {
- if (i == args.length-1) {
- throw new IllegalArgumentException("must specify a value for -javaHome");
- }
- i++;//skip argument value
- continue;
- }
-
- argList.add(args[i]);
- }
- return argList;
- }
-
- private static String extractJavaPath(String[] args) {
- String javaHome = null;
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-javaHome")) {
- if (i == args.length-1) {
- throw new IllegalArgumentException("must specify a value for -javaHome");
- }
- javaHome = args[i+1];
- break;
- }
- }
- if (javaHome == null) {
- javaHome = System.getenv("JAVA_HOME");
- }
- if (javaHome != null) {
- Path jPath = Paths.get(javaHome).resolve("bin").resolve("java");
- return ProcessUtils.escapeCommandLine(
- jPath.toAbsolutePath().toString());
- }
- return "java";
- }
- private static List<String> extractJVMArgs(String[] args) {
- List<String> jvmArgs = new ArrayList<>();
- boolean foundHeadlessOption = false;
- for (String arg : args) {
- if (arg.startsWith("-J")) {
- jvmArgs.add("-" + arg.substring(2));
- }
- if (arg.contains("java.awt.headless")) {
- foundHeadlessOption = true;
- }
- }
- //if user has already specified headless...don't modify
- if (! foundHeadlessOption) {
- jvmArgs.add("-Djava.awt.headless=true");
- }
-
- return jvmArgs;
- }
-
private class ForkedProcess {
private Thread SHUTDOWN_HOOK = null;
@@ -326,9 +278,7 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
throw new RuntimeException("couldn't read from status file after "+elapsed +" millis");
}
-
private void close() {
-
try {
if (toForked != null) {
toForked.writeByte(ServerStatus.DIRECTIVES.SHUTDOWN.getByte());
@@ -357,7 +307,6 @@ public class TikaServerWatchDog implements Callable<WatchDogResult> {
LOG.warn("problem deleting forked process status file", e);
}
}
-
}
private Process startProcess(int numRestarts, Path forkedStatusFile) throws IOException {
diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
index 6338dc4..ce437d7 100644
--- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
+++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/EmitterResource.java
@@ -223,7 +223,7 @@ public class EmitterResource {
if (t.getOnParseException() == FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP) {
shouldEmit = false;
}
- LOG.warn("fetchKey ({}) container parse exception ({})",
+ LOG.warn("fetchKey ({}) caught container parse exception ({})",
t.getFetchKey().getKey(), stack);
}
@@ -231,7 +231,7 @@ public class EmitterResource {
Metadata m = metadataList.get(i);
String embeddedStack = m.get(TikaCoreProperties.EMBEDDED_EXCEPTION);
if (embeddedStack != null) {
- LOG.warn("fetchKey ({}) embedded parse exception ({})",
+ LOG.warn("fetchKey ({}) caught embedded parse exception ({})",
t.getFetchKey().getKey(), embeddedStack);
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java
index a27f74b..fff4c0e 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/IntegrationTestBase.java
@@ -16,17 +16,30 @@
*/
package org.apache.tika.server.core;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.cxf.common.logging.LogUtils;
+import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.tika.TikaTest;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.Permission;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
public class IntegrationTestBase extends TikaTest {
@@ -41,6 +54,7 @@ public class IntegrationTestBase extends TikaTest {
static final String STATUS_PATH = "/status";
static final long MAX_WAIT_MS = 60000;
+ private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestBase.class);
//running into conflicts on 9998 with the CXFTestBase tests
//TODO: figure out why?!
@@ -51,6 +65,7 @@ public class IntegrationTestBase extends TikaTest {
private SecurityManager existingSecurityManager = null;
static Path LOG_FILE;
+ static Path STREAMS_DIR;
@BeforeClass
@@ -59,12 +74,13 @@ public class IntegrationTestBase extends TikaTest {
LOG_FILE = Files.createTempFile("tika-server-integration", ".xml");
Files.copy(TikaServerIntegrationTest.class.getResourceAsStream("/logging/log4j_forked.xml"),
LOG_FILE, StandardCopyOption.REPLACE_EXISTING);
+ STREAMS_DIR = Files.createTempDirectory("tika-server-integration");
}
@Before
public void setUp() throws Exception {
existingSecurityManager = System.getSecurityManager();
- System.setSecurityManager(new SecurityManager() {
+/* System.setSecurityManager(new SecurityManager() {
@Override
public void checkExit(int status) {
super.checkExit(status);
@@ -78,12 +94,13 @@ public class IntegrationTestBase extends TikaTest {
public void checkPermission(Permission perm, Object context) {
// all ok
}
- });
+ });*/
}
@AfterClass
public static void staticTearDown() throws Exception {
Files.delete(LOG_FILE);
+ FileUtils.deleteDirectory(STREAMS_DIR.toFile());
}
@After
@@ -101,4 +118,44 @@ public class IntegrationTestBase extends TikaTest {
return status;
}
}
+
+ public Process startProcess(String[] extraArgs) throws IOException {
+ String[] base = new String[] {
+ "java", "-cp", System.getProperty("java.class.path"),
+ "org.apache.tika.server.core.TikaServerCli",
+ };
+ List<String> args = new ArrayList<>(Arrays.asList(base));
+ args.addAll(Arrays.asList(extraArgs));
+ ProcessBuilder pb = new ProcessBuilder(args);
+ pb.redirectInput(Files.createTempFile(STREAMS_DIR, "tika-stream-out", ".log").toFile());
+ pb.redirectError(Files.createTempFile(STREAMS_DIR, "tika-stream-err", ".log").toFile());
+ return pb.start();
+ }
+
+ void awaitServerStartup() throws Exception {
+ Instant started = Instant.now();
+ long elapsed = Duration.between(started, Instant.now()).toMillis();
+ WebClient client = WebClient.create(endPoint + "/").accept("text/html");
+ while (elapsed < MAX_WAIT_MS) {
+ try {
+ Response response = client.get();
+ if (response.getStatus() == 200) {
+ elapsed = Duration.between(started, Instant.now()).toMillis();
+ LOG.info("client observes server successfully started after " +
+ elapsed + " ms");
+ return;
+ }
+ LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
+
+ } catch (javax.ws.rs.ProcessingException e) {
+ LOG.debug("tika test client failed to connect to server", e);
+ }
+
+ Thread.sleep(100);
+ elapsed = Duration.between(started, Instant.now()).toMillis();
+ }
+ throw new TimeoutException("couldn't connect to server after " +
+ elapsed + " ms");
+ }
+
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java
index e1812dc..d99490c 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerAsyncIntegrationTest.java
@@ -175,33 +175,6 @@ public class TikaServerAsyncIntegrationTest extends IntegrationTestBase {
return TMP_OUTPUT_DIR.toFile().listFiles().length;
}
-
- private void awaitServerStartup() throws Exception {
- Instant started = Instant.now();
- long elapsed = Duration.between(started, Instant.now()).toMillis();
- WebClient client = WebClient.create(endPoint+"/tika").accept("text/plain");
- while (elapsed < MAX_WAIT_MS) {
- try {
- Response response = client.get();
- if (response.getStatus() == 200) {
- elapsed = Duration.between(started, Instant.now()).toMillis();
- LOG.info("client observes server successfully started after " +
- elapsed+ " ms");
- return;
- }
- LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
-
- } catch (ProcessingException e) {
- LOG.debug("tika test client failed to connect to server", e);
- }
-
- Thread.sleep(100);
- elapsed = Duration.between(started, Instant.now()).toMillis();
- }
- throw new TimeoutException("couldn't connect to server after " +
- elapsed + " ms");
- }
-
private JsonNode sendAsync(List<String> fileNames) throws Exception {
awaitServerStartup();
List<FetchEmitTuple> tuples = new ArrayList<>();
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java
index ac9b6c3..9504c60 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerConfigTest.java
@@ -4,17 +4,25 @@ import org.apache.tika.config.TikaConfigTest;
import org.junit.Test;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TikaServerConfigTest {
@Test
public void testBasic() throws Exception {
+ Set<String> settings = new HashSet<>();
TikaServerConfig config = TikaServerConfig.load(
- TikaConfigTest.class.getResourceAsStream("/configs/tika-config-server.xml"));
+ TikaConfigTest.class.getResourceAsStream(
+ "/configs/tika-config-server.xml"), settings);
assertEquals(-1, config.getMaxRestarts());
assertEquals(54321, config.getTaskTimeoutMillis());
assertEquals(true, config.isEnableUnsecureFeatures());
+
+ assertTrue(settings.contains("taskTimeoutMillis"));
+ assertTrue(settings.contains("enableUnsecureFeatures"));
}
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
index 80c6759..15dfea6 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerEmitterIntegrationTest.java
@@ -26,6 +26,8 @@ import org.apache.tika.metadata.serialization.JsonFetchEmitTuple;
import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetchiterator.FetchEmitTuple;
+import org.apache.tika.utils.ProcessUtils;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -57,8 +59,8 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
private static Path TMP_DIR;
private static Path TMP_OUTPUT_DIR;
- private static String TIKA_CONFIG_XML;
private static Path TIKA_CONFIG;
+ private static Path TIKA_CONFIG_TIMEOUT;
private static final String EMITTER_NAME = "fse";
private static final String FETCHER_NAME = "fsf";
@@ -83,8 +85,9 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
inputDir.resolve(mockFile));
}
TIKA_CONFIG = TMP_DIR.resolve("tika-config.xml");
+ TIKA_CONFIG_TIMEOUT = TMP_DIR.resolve("tika-config-timeout.xml");
- TIKA_CONFIG_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
+ String xml1 = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
"<properties>" +
"<fetchers>" +
"<fetcher class=\"org.apache.tika.pipes.fetcher.FileSystemFetcher\">" +
@@ -103,9 +106,24 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
"</params>" +
"</emitter>" +
"</emitters>" +
+ "<server>" +
+ "<enableUnsecureFeatures>true</enableUnsecureFeatures>" +
+ "<port>9999</port>"+
+ "<endpoints>"+
+ "<endpoint>emit</endpoint>"+
+ "<endpoint>status</endpoint>"+
+ "</endpoints>";
+ String xml2 = "</server>"+
"</properties>";
- FileUtils.write(TIKA_CONFIG.toFile(), TIKA_CONFIG_XML, UTF_8);
+ String tikaConfigXML = xml1+xml2;
+
+ FileUtils.write(TIKA_CONFIG.toFile(), tikaConfigXML, UTF_8);
+
+ String tikaConfigTimeoutXML = xml1+
+ "<taskTimeoutMillis>10000</taskTimeoutMillis>"+xml2;
+ FileUtils.write(TIKA_CONFIG_TIMEOUT.toFile(), tikaConfigTimeoutXML, UTF_8);
+
}
@AfterClass
@@ -113,8 +131,14 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
FileUtils.deleteDirectory(TMP_DIR.toFile());
}
+ @After
+ public void tear() throws Exception {
+ Thread.sleep(500);
+ }
+
@Before
public void setUpEachTest() throws Exception {
+
for (String problemFile : FILES) {
Path targ = TMP_OUTPUT_DIR.resolve(problemFile + ".json");
@@ -127,49 +151,28 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
@Test
public void testBasic() throws Exception {
-
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess( new String[]{ "-config",
+ ProcessUtils.escapeCommandLine(TIKA_CONFIG.toAbsolutePath().toString())});
JsonNode node = testOne("hello_world.xml", true);
assertEquals("ok", node.get("status").asText());
} catch (Exception e) {
fail("shouldn't have an exception" + e.getMessage());
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testNPEDefault() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess( new String[]{ "-config",
+ ProcessUtils.escapeCommandLine(TIKA_CONFIG.toAbsolutePath().toString())});
JsonNode node = testOne("null_pointer.xml", true);
assertEquals("ok", node.get("status").asText());
assertContains("java.lang.NullPointerException",
@@ -177,28 +180,19 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
} catch (Exception e) {
fail("shouldn't have an exception" + e.getMessage());
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testNPESkip() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess( new String[]{ "-config",
+ ProcessUtils.escapeCommandLine(TIKA_CONFIG.toAbsolutePath().toString())});
JsonNode node = testOne("null_pointer.xml", false,
FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP);
assertEquals("ok", node.get("status").asText());
@@ -207,53 +201,33 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
} catch (Exception e) {
fail("shouldn't have an exception" + e.getMessage());
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test(expected = ProcessingException.class)
public void testSystemExit() throws Exception {
-
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess( new String[]{ "-config",
+ ProcessUtils.escapeCommandLine(TIKA_CONFIG.toAbsolutePath().toString())});
testOne("system_exit.xml", false);
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testOOM() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-JXmx128m",
- "-maxFiles", "2000",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess( new String[]{ "-config",
+ ProcessUtils.escapeCommandLine(TIKA_CONFIG.toAbsolutePath().toString())});
JsonNode response = testOne("fake_oom.xml", false);
assertContains("oom message", response.get("parse_error").asText());
} catch (ProcessingException e) {
@@ -261,60 +235,28 @@ public class TikaServerEmitterIntegrationTest extends IntegrationTestBase {
// TODO add more of a delay to server shutdown to ensure message is sent
// before shutdown.
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test(expected = ProcessingException.class)
public void testTimeout() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-enableUnsecureFeatures",
- "-JXmx128m",
- "-taskTimeoutMillis", "2000", "-taskPulseMillis", "100",
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "basic-",
- "-config", TIKA_CONFIG.toAbsolutePath().toString()
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess( new String[]{ "-config",
+ ProcessUtils.escapeCommandLine(TIKA_CONFIG_TIMEOUT.toAbsolutePath().toString())});
JsonNode response = testOne("heavy_hang_30000.xml", false);
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
- private void awaitServerStartup() throws Exception {
- Instant started = Instant.now();
- long elapsed = Duration.between(started, Instant.now()).toMillis();
- WebClient client = WebClient.create(endPoint + "/tika").accept("text/plain");
- while (elapsed < MAX_WAIT_MS) {
- try {
- Response response = client.get();
- if (response.getStatus() == 200) {
- elapsed = Duration.between(started, Instant.now()).toMillis();
- LOG.info("client observes server successfully started after " +
- elapsed + " ms");
- return;
- }
- LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
-
- } catch (javax.ws.rs.ProcessingException e) {
- LOG.debug("tika test client failed to connect to server", e);
- }
- Thread.sleep(100);
- elapsed = Duration.between(started, Instant.now()).toMillis();
- }
- throw new TimeoutException("couldn't connect to server after " +
- elapsed + " ms");
- }
private JsonNode testOne(String fileName, boolean shouldFileExist) throws Exception {
return testOne(fileName, shouldFileExist, FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT);
}
diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
index b9d58d7..a191cb0 100644
--- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
+++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaServerIntegrationTest.java
@@ -23,6 +23,7 @@ import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.serialization.JsonMetadataList;
import org.apache.tika.utils.ProcessUtils;
+import org.junit.After;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@@ -46,6 +47,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -57,146 +59,118 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
private static final Logger LOG = LoggerFactory.getLogger(TikaServerIntegrationTest.class);
+ @After
+ public void tearDown() throws Exception {
+ //wait just a bit for the servers to shutdown
+ Thread.sleep(500);
+ }
@Test
public void testBasic() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-c", getConfig("tika-config-server-basic.xml")
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-basic.xml")});
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testOOM() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-JXmx256m",
- "-p", INTEGRATION_TEST_PORT,
- "-pingPulseMillis", "100",
- "-tmpFilePrefix", "tika-server-oom"
+ Process p = null;
+ try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-basic.xml")});
- });
- }
- };
- serverThread.start();
- awaitServerStartup();
+ awaitServerStartup();
- Response response = null;
- try {
- response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_OOM));
- } catch (Exception e) {
- //oom may or may not cause an exception depending
- //on the timing
- }
- //give some time for the server to crash/terminate itself
- Thread.sleep(2000);
- try {
+ Response response = null;
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_OOM));
+ } catch (Exception e) {
+ //oom may or may not cause an exception depending
+ //on the timing
+ }
+ //give some time for the server to crash/terminate itself
+ Thread.sleep(2000);
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testSameServerIdAfterOOM() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-JXmx256m",
- "-p", INTEGRATION_TEST_PORT,
- "-pingPulseMillis", "100",
- "-status",
- "-tmpFilePrefix", "tika-server-oom"
- });
- }
- };
- serverThread.start();
- awaitServerStartup();
- String serverId = getServerId();
- Response response = null;
- try {
- response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_OOM));
- } catch (Exception e) {
- //oom may or may not cause an exception depending
- //on the timing
- }
- //give some time for the server to crash/terminate itself
- Thread.sleep(2000);
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-basic.xml")});
+ awaitServerStartup();
+ String serverId = getServerId();
+ Response response = null;
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_OOM));
+ } catch (Exception e) {
+ //oom may or may not cause an exception depending
+ //on the timing
+ }
+ //give some time for the server to crash/terminate itself
+ Thread.sleep(2000);
testBaseline();
assertEquals(serverId, getServerId());
assertEquals(1, getNumRestarts());
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testSameDeclaredServerIdAfterOOM() throws Exception {
String serverId = "qwertyuiop";
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-JXmx256m",
- "-p", INTEGRATION_TEST_PORT,
- "-pingPulseMillis", "100",
- "-status",
- "-id",
- serverId,
- "-tmpFilePrefix", "tika-server-oom"
-
- });
- }
- };
- serverThread.start();
- awaitServerStartup();
- Response response = null;
- try {
- response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_OOM));
- } catch (Exception e) {
- //oom may or may not cause an exception depending
- //on the timing
- }
- //give some time for the server to crash/terminate itself
- Thread.sleep(2000);
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-basic.xml"),
+ "-id", serverId});
+ awaitServerStartup();
+ Response response = null;
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_OOM));
+ } catch (Exception e) {
+ //oom may or may not cause an exception depending
+ //on the timing
+ }
+ //give some time for the server to crash/terminate itself
+ Thread.sleep(2000);
testBaseline();
assertEquals(serverId, getServerId());
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@@ -225,136 +199,110 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
@Test
public void testSystemExit() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-p", INTEGRATION_TEST_PORT,
- "-tmpFilePrefix", "tika-server-systemexit"
+ Process p = null;
+ try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-basic.xml")});
- });
+ awaitServerStartup();
+ Response response = null;
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_SYSTEM_EXIT));
+ } catch (Exception e) {
+ //sys exit causes catchable problems for the client
}
- };
- serverThread.start();
- awaitServerStartup();
- Response response = null;
- try {
- response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_SYSTEM_EXIT));
- } catch (Exception e) {
- //sys exit causes catchable problems for the client
- }
- //give some time for the server to crash/terminate itself
- Thread.sleep(2000);
- try {
+ //give some time for the server to crash/terminate itself
+ Thread.sleep(2000);
+
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
+
}
}
@Test
public void testTimeoutOk() throws Exception {
- //test that there's enough time for this file.
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-p", INTEGRATION_TEST_PORT,
- "-taskTimeoutMillis", "10000", "-taskPulseMillis", "500",
- "-pingPulseMillis", "500",
- "-tmpFilePrefix", "tika-server-timeoutok"
-
- });
- }
- };
- serverThread.start();
- awaitServerStartup();
- Response response = null;
- try {
- response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_HEAVY_HANG_SHORT));
- } catch (Exception e) {
- //potential exception depending on timing
- }
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-timeout-10000.xml")});
+ awaitServerStartup();
+ Response response = null;
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_HEAVY_HANG_SHORT));
+ } catch (Exception e) {
+ //potential exception depending on timing
+ }
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test(timeout = 60000)
public void testTimeout() throws Exception {
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-p", INTEGRATION_TEST_PORT,
- "-taskTimeoutMillis", "10000", "-taskPulseMillis", "100",
- "-pingPulseMillis", "100",
- "-tmpFilePrefix", "tika-server-timeout"
-
- });
- }
- };
- serverThread.start();
- awaitServerStartup();
- Response response = null;
- try {
- response = WebClient
- .create(endPoint + META_PATH)
- .accept("application/json")
- .put(ClassLoader
- .getSystemResourceAsStream(TEST_HEAVY_HANG));
- } catch (Exception e) {
- //catchable exception when server shuts down.
- }
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-timeout-10000.xml")});
+ awaitServerStartup();
+ Response response = null;
+ try {
+ response = WebClient
+ .create(endPoint + META_PATH)
+ .accept("application/json")
+ .put(ClassLoader
+ .getSystemResourceAsStream(TEST_HEAVY_HANG));
+ } catch (Exception e) {
+ //catchable exception when server shuts down.
+ }
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
public void testBadJVMArgs() throws Exception {
- final AtomicInteger i = new AtomicInteger();
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-c", getConfig("tika-config-server-badjvmargs.xml"),
- });
+
+ Process p = null;
+ try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-badjvmargs.xml"),
+ });
+
+ boolean finished = p.waitFor(10000, TimeUnit.MILLISECONDS);
+ if (! finished) {
+ fail("should have completed by now");
}
- };
- serverThread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- i.set(((MyExitException) e).getStatus());
+ assertEquals(255, p.exitValue());
+ } finally {
+ if (p != null) {
+ p.destroyForcibly();
}
- });
- serverThread.start();
- serverThread.join(30000);
-
- assertEquals(-1, i.get());
+ }
}
private String getConfig(String configName) {
try {
return ProcessUtils.escapeCommandLine(Paths.get(TikaServerIntegrationTest.class.
- getResource("/configs/"+configName).toURI()).toAbsolutePath().toString());
+ getResource("/configs/" + configName).toURI()).toAbsolutePath().toString());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
@@ -362,22 +310,10 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
@Test
public void testStdErrOutBasic() throws Exception {
- final AtomicInteger i = new AtomicInteger();
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-p", INTEGRATION_TEST_PORT,
- "-taskTimeoutMillis", "60000", "-taskPulseMillis", "500",
- "-pingPulseMillis", "100",
- "-tmpFilePrefix", "tika-server-stderr"
-
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-timeout-10000.xml")});
awaitServerStartup();
Response response = WebClient
@@ -391,27 +327,19 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
assertContains("quick brown fox", metadataList.get(0).get("X-TIKA:content"));
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
}
}
@Test
@Ignore("This works, but prints too much junk to the console. Figure out how to gobble/redirect.")
public void testStaticStdErrOutBasic() throws Exception {
- final AtomicInteger i = new AtomicInteger();
- Thread serverThread = new Thread() {
- @Override
- public void run() {
- TikaServerCli.main(
- new String[]{
- "-p", INTEGRATION_TEST_PORT,
- "-taskTimeoutMillis", "60000", "-taskPulseMillis", "500",
- "-pingPulseMillis", "100"
- });
- }
- };
- serverThread.start();
+ Process p = null;
try {
+ p = startProcess(new String[]{"-config",
+ getConfig("tika-config-server-timeout-10000.xml")});
awaitServerStartup();
Response response = WebClient
@@ -425,11 +353,15 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
assertContains("quick brown fox", metadataList.get(0).get("X-TIKA:content"));
testBaseline();
} finally {
- serverThread.interrupt();
+ if (p != null) {
+ p.destroyForcibly();
+ }
+
}
}
+ @Ignore("TODO needs to write dynamic config file w logfile location")
@Test
public void testStdErrOutLogging() throws Exception {
final AtomicInteger i = new AtomicInteger();
@@ -466,36 +398,6 @@ public class TikaServerIntegrationTest extends IntegrationTestBase {
}
}
- @Test
- public void testEmitterSysExit() throws Exception {
-
- }
-
- private void awaitServerStartup() throws Exception {
- Instant started = Instant.now();
- long elapsed = Duration.between(started, Instant.now()).toMillis();
- WebClient client = WebClient.create(endPoint + "/").accept("text/html");
- while (elapsed < MAX_WAIT_MS) {
- try {
- Response response = client.get();
- if (response.getStatus() == 200) {
- elapsed = Duration.between(started, Instant.now()).toMillis();
- LOG.info("client observes server successfully started after " +
- elapsed + " ms");
- return;
- }
- LOG.debug("tika test client failed to connect to server with status: {}", response.getStatus());
-
- } catch (javax.ws.rs.ProcessingException e) {
- LOG.debug("tika test client failed to connect to server", e);
- }
-
- Thread.sleep(100);
- elapsed = Duration.between(started, Instant.now()).toMillis();
- }
- throw new TimeoutException("couldn't connect to server after " +
- elapsed + " ms");
- }
@Test
@Ignore("turn this into a real test")
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
index 42a2c58..d5c4f73 100644
--- a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
@@ -25,6 +25,7 @@
</forkedJVMArgs>
<endpoints>
<endpoint>rmeta</endpoint>
+ <endpoint>status</endpoint>
</endpoints>
</server>
</properties>
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-emitter.xml
similarity index 88%
copy from tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
copy to tika-server/tika-server-core/src/test/resources/configs/tika-config-server-emitter.xml
index 42a2c58..9f30f20 100644
--- a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-basic.xml
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-emitter.xml
@@ -23,8 +23,11 @@
<forkedJVMArgs>
<arg>-Xmx512m</arg>
</forkedJVMArgs>
+ <enableUnsecure
<endpoints>
- <endpoint>rmeta</endpoint>
+ <endpoint>emit</endpoint>
+ <endpoint>async</endpoint>
+ <endpoint>status</endpoint>
</endpoints>
</server>
</properties>
diff --git a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml
index 78ec922..6ff6280 100644
--- a/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml
+++ b/tika-server/tika-server-core/src/test/resources/configs/tika-config-server-timeout-10000.xml
@@ -17,8 +17,10 @@
-->
<properties>
<server>
- <taskTimeoutMillis>120000</taskTimeoutMillis>
- <maxFiles>1000</maxFiles>
+ <port>9999</port>
+ <taskTimeoutMillis>10000</taskTimeoutMillis>
+ <pingTimeoutMillis>30000</pingTimeoutMillis>
+ <pingPulseMillis>100</pingPulseMillis>
<forkedJVMArgs>
<arg>-Xmx512m</arg>
</forkedJVMArgs>
diff --git a/tika-server/tika-server-core/src/test/resources/log4j.properties b/tika-server/tika-server-core/src/test/resources/log4j.properties
index c3676a2..d39a651 100644
--- a/tika-server/tika-server-core/src/test/resources/log4j.properties
+++ b/tika-server/tika-server-core/src/test/resources/log4j.properties
@@ -16,9 +16,13 @@
#info,debug, error,fatal ...
log4j.rootLogger=warn,stderr
+#turn off cxf client logging
+#for development, you can downgrade this to warn
+log4j.logger.org.apache.cxf=fatal,stderr
+log4j.additivity.org.apache.cxf=false
+
#console
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.Target=System.err
-
log4j.appender.stderr.layout.ConversionPattern= %-5p %d [%t] (%F:%L) - %m%n