You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2021/06/02 13:49:30 UTC
[tika] branch main updated (42f6e44 -> 07fbec4)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 42f6e44 fix for TIKA-3400 contributed by kamaci (#441)
new 075e594 update tika-app for checkstyle
new 1028ccf Merge remote-tracking branch 'origin/main' into main
new 07fbec4 TIKA-3424 -- tika-app should log to stderr
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tika-app/pom.xml | 5 -
.../batch/builders/AppParserFactoryBuilder.java | 23 +-
.../apache/tika/cli/BatchCommandLineBuilder.java | 37 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 855 ++++++++++-----------
.../apache/tika/gui/ParsingTransferHandler.java | 68 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 200 ++---
tika-app/src/main/resources/log4j2.properties | 28 -
.../src/main/resources/log4j2.xml | 16 +-
.../tika/cli/TikaCLIBatchCommandLineTest.java | 15 +-
.../tika/cli/TikaCLIBatchIntegrationTest.java | 56 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 243 +++---
.../tika/extractor/TestEmbeddedDocumentUtil.java | 19 +-
.../resources/log4j2_batch_process_test.properties | 1 +
tika-batch/pom.xml | 15 +-
tika-batch/src/test/resources/log4j2.properties | 28 -
.../src/test/resources/log4j2.xml | 16 +-
tika-bundles/tika-bundle-standard/pom.xml | 4 +-
.../tika-eval-app/src/main/resources/log4j2.xml | 16 +-
.../src/main/resources/log4j.properties | 20 -
tika-fuzzing/pom.xml | 4 -
tika-fuzzing/src/main/resources/log4j2.properties | 28 -
.../src/main/resources/log4j2.xml | 14 +-
.../src/test/resources/log4j2.properties | 29 -
.../src/main/resources/log4j2.properties | 28 -
.../src/main/resources/log4j2.xml | 16 +-
tika-parent/pom.xml | 13 +-
.../tika-parser-microsoft-module/pom.xml | 14 +-
.../tika/parser/microsoft/WordParserTest.java | 14 +-
.../tika-parser-pdf-module/pom.xml | 6 -
.../tika-parsers-standard-package/pom.xml | 17 +
.../src/test/resources/log4j2.properties | 28 -
.../src/test/resources/log4j2.xml | 16 +-
.../src/main/resources/log4j2.properties | 29 -
.../src/main/resources/log4j2.xml | 16 +-
.../src/test/resources/log4j2.properties | 29 -
.../src/test/resources/log4j2.xml | 16 +-
tika-server/tika-server-core/pom.xml | 5 -
.../src/test/resources/log4j2.properties | 33 -
.../tika-server-core/src/test/resources/log4j2.xml | 16 +-
.../src/main/resources/log4j2.properties | 26 -
.../src/main/resources/log4j2.xml | 16 +-
.../src/test/resources/log4j2.properties | 25 -
.../src/test/resources/log4j2.xml | 16 +-
43 files changed, 850 insertions(+), 1269 deletions(-)
delete mode 100644 tika-app/src/main/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-app/src/main/resources/log4j2.xml (74%)
delete mode 100644 tika-batch/src/test/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-batch/src/test/resources/log4j2.xml (74%)
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-eval/tika-eval-app/src/main/resources/log4j2.xml (74%)
delete mode 100644 tika-eval/tika-eval-core/src/main/resources/log4j.properties
delete mode 100644 tika-fuzzing/src/main/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-fuzzing/src/main/resources/log4j2.xml (76%)
delete mode 100644 tika-langdetect/tika-langdetect-mitll-text/src/test/resources/log4j2.properties
delete mode 100644 tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.xml (74%)
delete mode 100644 tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.xml (74%)
delete mode 100644 tika-server/tika-server-client/src/main/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-server/tika-server-client/src/main/resources/log4j2.xml (74%)
delete mode 100644 tika-server/tika-server-client/src/test/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-server/tika-server-client/src/test/resources/log4j2.xml (74%)
delete mode 100644 tika-server/tika-server-core/src/test/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-server/tika-server-core/src/test/resources/log4j2.xml (74%)
delete mode 100644 tika-server/tika-server-standard/src/main/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-server/tika-server-standard/src/main/resources/log4j2.xml (74%)
delete mode 100644 tika-server/tika-server-standard/src/test/resources/log4j2.properties
copy tika-core/src/main/resources/pipes-fork-server-default-log4j2.xml => tika-server/tika-server-standard/src/test/resources/log4j2.xml (74%)
[tika] 03/03: TIKA-3424 -- tika-app should log to stderr
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 07fbec45a6aff0e3eae1dcbb8417dcfb124b77a7
Author: tallison <ta...@apache.org>
AuthorDate: Wed Jun 2 09:49:15 2021 -0400
TIKA-3424 -- tika-app should log to stderr
---
tika-app/pom.xml | 5 ----
tika-app/src/main/resources/log4j2.properties | 28 ------------------
tika-app/src/main/resources/log4j2.xml | 32 +++++++++++++++++++++
tika-batch/pom.xml | 15 ++--------
tika-batch/src/test/resources/log4j2.properties | 28 ------------------
tika-batch/src/test/resources/log4j2.xml | 32 +++++++++++++++++++++
tika-bundles/tika-bundle-standard/pom.xml | 4 ++-
.../tika-eval-app/src/main/resources/log4j2.xml | 32 +++++++++++++++++++++
.../src/main/resources/log4j.properties | 20 -------------
tika-fuzzing/pom.xml | 4 ---
tika-fuzzing/src/main/resources/log4j2.properties | 28 ------------------
tika-fuzzing/src/main/resources/log4j2.xml | 32 +++++++++++++++++++++
.../src/test/resources/log4j2.properties | 29 -------------------
.../src/main/resources/log4j2.properties | 28 ------------------
.../src/main/resources/log4j2.xml | 32 +++++++++++++++++++++
tika-parent/pom.xml | 11 --------
.../tika-parser-microsoft-module/pom.xml | 14 +++++----
.../tika/parser/microsoft/WordParserTest.java | 14 +++++----
.../tika-parser-pdf-module/pom.xml | 6 ----
.../tika-parsers-standard-package/pom.xml | 17 +++++++++++
.../src/test/resources/log4j2.properties | 28 ------------------
.../src/test/resources/log4j2.xml | 32 +++++++++++++++++++++
.../src/main/resources/log4j2.properties | 29 -------------------
.../src/main/resources/log4j2.xml | 32 +++++++++++++++++++++
.../src/test/resources/log4j2.properties | 29 -------------------
.../src/test/resources/log4j2.xml | 32 +++++++++++++++++++++
tika-server/tika-server-core/pom.xml | 5 ----
.../src/test/resources/log4j2.properties | 33 ----------------------
.../tika-server-core/src/test/resources/log4j2.xml | 32 +++++++++++++++++++++
.../src/main/resources/log4j2.properties | 26 -----------------
.../src/main/resources/log4j2.xml | 32 +++++++++++++++++++++
.../src/test/resources/log4j2.properties | 25 ----------------
.../src/test/resources/log4j2.xml | 32 +++++++++++++++++++++
33 files changed, 393 insertions(+), 385 deletions(-)
diff --git a/tika-app/pom.xml b/tika-app/pom.xml
index 6ac21fb..189515a 100644
--- a/tika-app/pom.xml
+++ b/tika-app/pom.xml
@@ -85,11 +85,6 @@
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- </dependency>
-
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
diff --git a/tika-app/src/main/resources/log4j2.properties b/tika-app/src/main/resources/log4j2.properties
deleted file mode 100644
index d17a4a1..0000000
--- a/tika-app/src/main/resources/log4j2.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
diff --git a/tika-app/src/main/resources/log4j2.xml b/tika-app/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-app/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-batch/pom.xml b/tika-batch/pom.xml
index 413e933..ed69253 100644
--- a/tika-batch/pom.xml
+++ b/tika-batch/pom.xml
@@ -69,21 +69,12 @@
<scope>test</scope>
</dependency>
- <!-- logging -->
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <version>${log4j2.version}</version>
- </dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
</dependency>
-
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git a/tika-batch/src/test/resources/log4j2.properties b/tika-batch/src/test/resources/log4j2.properties
deleted file mode 100644
index dca624f..0000000
--- a/tika-batch/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-status=error
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=error
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p %m%n
-rootLogger.level=ERROR
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
diff --git a/tika-batch/src/test/resources/log4j2.xml b/tika-batch/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-batch/src/test/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-bundles/tika-bundle-standard/pom.xml b/tika-bundles/tika-bundle-standard/pom.xml
index 8c78dfe..fce99c8 100644
--- a/tika-bundles/tika-bundle-standard/pom.xml
+++ b/tika-bundles/tika-bundle-standard/pom.xml
@@ -144,6 +144,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
@@ -267,8 +268,9 @@
com.adobe.xmp;resolution:=optional,
com.adobe.xmp.impl;resolution:=optional,
com.adobe.xmp.options;resolution:=optional,
- com.dd.plist;resolution:=optional,
com.adobe.xmp.properties;resolution:=optional,
+ com.apple.eawt;resolution:=optional,
+ com.dd.plist;resolution:=optional,
com.github.luben.zstd;resolution:=optional,
com.github.jaiimageio.*;resolution:=optional,
com.google.common.util.concurrent.internal;resolution:=optional,
diff --git a/tika-eval/tika-eval-app/src/main/resources/log4j2.xml b/tika-eval/tika-eval-app/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-eval/tika-eval-app/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-eval/tika-eval-core/src/main/resources/log4j.properties b/tika-eval/tika-eval-core/src/main/resources/log4j.properties
deleted file mode 100644
index 223e3ce..0000000
--- a/tika-eval/tika-eval-core/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-log4j.rootLogger=INFO,A1
-log4j.appender.A1=org.apache.log4j.ConsoleAppender
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
diff --git a/tika-fuzzing/pom.xml b/tika-fuzzing/pom.xml
index 3a276e1..9a280f2 100644
--- a/tika-fuzzing/pom.xml
+++ b/tika-fuzzing/pom.xml
@@ -58,10 +58,6 @@
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- </dependency>
<!-- test -->
<dependency>
<groupId>org.apache.tika</groupId>
diff --git a/tika-fuzzing/src/main/resources/log4j2.properties b/tika-fuzzing/src/main/resources/log4j2.properties
deleted file mode 100644
index 21c2ff2..0000000
--- a/tika-fuzzing/src/main/resources/log4j2.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#info,debug, error,fatal ...
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
diff --git a/tika-fuzzing/src/main/resources/log4j2.xml b/tika-fuzzing/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..513b667
--- /dev/null
+++ b/tika-fuzzing/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/tika-langdetect/tika-langdetect-mitll-text/src/test/resources/log4j2.properties b/tika-langdetect/tika-langdetect-mitll-text/src/test/resources/log4j2.properties
deleted file mode 100644
index 835e1ae..0000000
--- a/tika-langdetect/tika-langdetect-mitll-text/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#info,debug, error,fatal ...
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDOUT
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%5p [%t] (%F:%L) - %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stdout
-rootLogger.appenderRef.stderr.ref=STDOUT
diff --git a/tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.properties b/tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.properties
deleted file mode 100644
index 97fe93a..0000000
--- a/tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDOUT
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%5p [%t] (%F:%L) - %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stdout
-rootLogger.appenderRef.stderr.ref=STDOUT
diff --git a/tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.xml b/tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-langdetect/tika-langdetect-test-commons/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index d8c409f..78a278f 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -393,21 +393,10 @@
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
- <dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
index 4a3c7df..28171a1 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/pom.xml
@@ -157,12 +157,16 @@
<artifactId>bcprov-jdk15on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
- <!-- https://mvnrepository.com/artifact/log4j/log4j -->
+ <!-- logging -->
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.17</version>
- <scope>test</scope>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
</dependencies>
<build>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
index 964f2ce..c0e6113 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
@@ -26,8 +26,10 @@ import java.util.Arrays;
import java.util.List;
import java.util.Locale;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.LoggerConfig;
import org.junit.Ignore;
import org.junit.Test;
import org.xml.sax.ContentHandler;
@@ -350,15 +352,17 @@ public class WordParserTest extends TikaTest {
@Test
public void testExceptions1() throws Exception {
XMLResult xml;
- Level logLevelStart = Logger.getRootLogger().getLevel();
- Logger.getRootLogger().setLevel(Level.ERROR);
+ LoggerContext ctx = (LoggerContext) LogManager.getContext();
+ LoggerConfig lc = ctx.getConfiguration().getRootLogger();
+ Level originalLevel = lc.getLevel();
+ lc.setLevel(Level.ERROR);
try {
xml = getXML("testException1.doc");
assertContains("total population", xml.xml);
xml = getXML("testException2.doc");
assertContains("electric charge", xml.xml);
} finally {
- Logger.getRootLogger().setLevel(logLevelStart);
+ lc.setLevel(originalLevel);
}
}
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
index 5c74cde..59b987c 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/pom.xml
@@ -44,12 +44,6 @@
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox-tools</artifactId>
<version>${pdfbox.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.pdfbox</groupId>
- <artifactId>pdfbox-debugger</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml
index 5d24f7e..7bd44fb 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/pom.xml
@@ -100,6 +100,17 @@
<groupId>${project.groupId}</groupId>
<artifactId>tika-parser-microsoft-module</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- swap in jcl-over-slf4j -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -121,6 +132,12 @@
<groupId>${project.groupId}</groupId>
<artifactId>tika-parser-pdf-module</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.properties b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.properties
deleted file mode 100644
index d17a4a1..0000000
--- a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
diff --git a/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.xml b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-server/tika-server-client/src/main/resources/log4j2.properties b/tika-server/tika-server-client/src/main/resources/log4j2.properties
deleted file mode 100644
index acb957b..0000000
--- a/tika-server/tika-server-client/src/main/resources/log4j2.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#info,debug, error,fatal ...
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
diff --git a/tika-server/tika-server-client/src/main/resources/log4j2.xml b/tika-server/tika-server-client/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-server/tika-server-client/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-server/tika-server-client/src/test/resources/log4j2.properties b/tika-server/tika-server-client/src/test/resources/log4j2.properties
deleted file mode 100644
index 035f3fb..0000000
--- a/tika-server/tika-server-client/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#info,debug, error,fatal ...
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=debug
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p %m%n
-rootLogger.level=debug
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
diff --git a/tika-server/tika-server-client/src/test/resources/log4j2.xml b/tika-server/tika-server-client/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-server/tika-server-client/src/test/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-server/tika-server-core/pom.xml b/tika-server/tika-server-core/pom.xml
index 0908172..60fea8f 100644
--- a/tika-server/tika-server-core/pom.xml
+++ b/tika-server/tika-server-core/pom.xml
@@ -159,11 +159,6 @@
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jul-to-slf4j</artifactId>
- </dependency>
-
- <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-core</artifactId>
<version>${project.version}</version>
diff --git a/tika-server/tika-server-core/src/test/resources/log4j2.properties b/tika-server/tika-server-core/src/test/resources/log4j2.properties
deleted file mode 100644
index e89a6e3..0000000
--- a/tika-server/tika-server-core/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p %d [%t] (%F:%L) - %m%n
-rootLogger.level=info
-rootLogger.appenderRefs=stderr
-rootLogger.appenderRef.stderr.ref=STDERR
-loggers=cxf
-logger.cxf.name=org.apache.cxf
-logger.cxf.level=fatal
-logger.cxf.appenderRefs=stderr
-log4j.appender.stderr.layout.ConversionPattern=
diff --git a/tika-server/tika-server-core/src/test/resources/log4j2.xml b/tika-server/tika-server-core/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-server/tika-server-core/src/test/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-server/tika-server-standard/src/main/resources/log4j2.properties b/tika-server/tika-server-standard/src/main/resources/log4j2.properties
deleted file mode 100644
index 36c879a..0000000
--- a/tika-server/tika-server-standard/src/main/resources/log4j2.properties
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#info,debug, error,fatal ...
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n
diff --git a/tika-server/tika-server-standard/src/main/resources/log4j2.xml b/tika-server/tika-server-standard/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-server/tika-server-standard/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/tika-server/tika-server-standard/src/test/resources/log4j2.properties b/tika-server/tika-server-standard/src/test/resources/log4j2.properties
deleted file mode 100644
index c7871eb..0000000
--- a/tika-server/tika-server-standard/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-status=info
-name=PropertiesConfig
-filters=threshold
-filter.threshold.type=ThresholdFilter
-filter.threshold.level=info
-appenders=console
-appender.console.type=Console
-appender.console.name=STDERR
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%-5p %d [%t] (%F:%L) - %m%n
diff --git a/tika-server/tika-server-standard/src/test/resources/log4j2.xml b/tika-server/tika-server-standard/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..c88e66e
--- /dev/null
+++ b/tika-server/tika-server-standard/src/test/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%-5p [%t] %d{HH:mm:ss,SSS} %c %m%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="info">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
\ No newline at end of file
[tika] 02/03: Merge remote-tracking branch 'origin/main' into main
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 1028ccfceea9e928fc344951fd1b3b923adda78f
Merge: 075e594 42f6e44
Author: tallison <ta...@apache.org>
AuthorDate: Wed Jun 2 09:46:49 2021 -0400
Merge remote-tracking branch 'origin/main' into main
.../main/java/org/apache/tika/eval/app/XMLErrorLogUpdater.java | 1 -
.../src/main/java/org/apache/tika/eval/app/db/Cols.java | 1 -
.../org/apache/tika/eval/app/io/ExtractReaderException.java | 2 +-
.../main/java/org/apache/tika/eval/app/io/XMLLogReader.java | 1 -
.../org/apache/tika/eval/app/tools/TopCommonTokenCounter.java | 7 ++-----
.../org/apache/tika/langdetect/tika/LanguageIdentifier.java | 2 +-
.../java/org/apache/tika/parser/sqlite3/SQLite3ParserTest.java | 2 +-
.../org/apache/tika/parser/recognition/AgeRecogniserTest.java | 2 +-
.../org/apache/tika/parser/pot/PooledTimeSeriesParser.java | 1 -
.../java/org/apache/tika/parser/iwork/PagesContentHandler.java | 2 +-
.../apache/tika/sax/boilerpipe/BoilerpipeContentHandler.java | 10 ----------
.../org/apache/tika/parser/microsoft/onenote/FileNode.java | 4 ++--
.../apache/tika/parser/microsoft/onenote/PropertyIDType.java | 2 +-
.../java/org/apache/tika/parser/strings/StringsParser.java | 1 -
.../test/java/org/apache/tika/server/core/TikaPipesTest.java | 9 ++++-----
.../tika/language/translate/JoshuaNetworkTranslator.java | 2 +-
16 files changed, 15 insertions(+), 34 deletions(-)
[tika] 01/03: update tika-app for checkstyle
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 075e594292dcf3be797981b666b268a20d6fb074
Author: tallison <ta...@apache.org>
AuthorDate: Fri May 28 14:18:35 2021 -0400
update tika-app for checkstyle
---
.../batch/builders/AppParserFactoryBuilder.java | 23 +-
.../apache/tika/cli/BatchCommandLineBuilder.java | 37 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 855 ++++++++++-----------
.../apache/tika/gui/ParsingTransferHandler.java | 68 +-
.../src/main/java/org/apache/tika/gui/TikaGUI.java | 200 ++---
.../tika/cli/TikaCLIBatchCommandLineTest.java | 15 +-
.../tika/cli/TikaCLIBatchIntegrationTest.java | 56 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 243 +++---
.../tika/extractor/TestEmbeddedDocumentUtil.java | 19 +-
.../resources/log4j2_batch_process_test.properties | 1 +
tika-parent/pom.xml | 2 +-
11 files changed, 722 insertions(+), 797 deletions(-)
diff --git a/tika-app/src/main/java/org/apache/tika/batch/builders/AppParserFactoryBuilder.java b/tika-app/src/main/java/org/apache/tika/batch/builders/AppParserFactoryBuilder.java
index 9c283d8..bfacbd9 100644
--- a/tika-app/src/main/java/org/apache/tika/batch/builders/AppParserFactoryBuilder.java
+++ b/tika-app/src/main/java/org/apache/tika/batch/builders/AppParserFactoryBuilder.java
@@ -20,6 +20,8 @@ package org.apache.tika.batch.builders;
import java.util.Locale;
import java.util.Map;
+import org.w3c.dom.Node;
+
import org.apache.tika.batch.DigestingAutoDetectParserFactory;
import org.apache.tika.batch.ParserFactory;
import org.apache.tika.parser.DigestingParser;
@@ -27,7 +29,6 @@ import org.apache.tika.parser.digestutils.BouncyCastleDigester;
import org.apache.tika.parser.digestutils.CommonsDigester;
import org.apache.tika.util.ClassLoaderUtil;
import org.apache.tika.util.XMLDOMUtil;
-import org.w3c.dom.Node;
public class AppParserFactoryBuilder implements IParserFactoryBuilder {
@@ -44,13 +45,13 @@ public class AppParserFactoryBuilder implements IParserFactoryBuilder {
} else if (bString.equals("false")) {
pf.setParseRecursively(false);
} else {
- throw new RuntimeException("parseRecursively must have value of \"true\" or \"false\": "+
- bString);
+ throw new RuntimeException(
+ "parseRecursively must have value of \"true\" or \"false\": " + bString);
}
}
if (pf instanceof DigestingAutoDetectParserFactory) {
DigestingParser.Digester d = buildDigester(localAttrs);
- ((DigestingAutoDetectParserFactory)pf).setDigester(d);
+ ((DigestingAutoDetectParserFactory) pf).setDigester(d);
}
return pf;
}
@@ -59,16 +60,16 @@ public class AppParserFactoryBuilder implements IParserFactoryBuilder {
String readLimitString = localAttrs.get("digestMarkLimit");
if (readLimitString == null) {
- throw new IllegalArgumentException("Must specify \"digestMarkLimit\" for "+
- "the DigestingAutoDetectParserFactory");
+ throw new IllegalArgumentException("Must specify \"digestMarkLimit\" for " +
+ "the DigestingAutoDetectParserFactory");
}
int readLimit = -1;
try {
readLimit = Integer.parseInt(readLimitString);
} catch (NumberFormatException e) {
- throw new IllegalArgumentException("Parameter \"digestMarkLimit\" must be a parseable int: "+
- readLimitString);
+ throw new IllegalArgumentException(
+ "Parameter \"digestMarkLimit\" must be a parseable int: " + readLimitString);
}
String digestString = localAttrs.get("digest");
try {
@@ -77,8 +78,10 @@ public class AppParserFactoryBuilder implements IParserFactoryBuilder {
try {
return new BouncyCastleDigester(readLimit, digestString);
} catch (IllegalArgumentException bcException) {
- throw new IllegalArgumentException("Tried both CommonsDigester ("+commonsException.getMessage()+
- ") and BouncyCastleDigester ("+bcException.getMessage()+")", bcException);
+ throw new IllegalArgumentException(
+ "Tried both CommonsDigester (" + commonsException.getMessage() +
+ ") and BouncyCastleDigester (" + bcException.getMessage() + ")",
+ bcException);
}
}
}
diff --git a/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java b/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java
index eb9545f..c3039cd 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/BatchCommandLineBuilder.java
@@ -18,8 +18,6 @@
package org.apache.tika.cli;
-import org.apache.commons.lang3.SystemUtils;
-
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -31,6 +29,8 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.SystemUtils;
+
/**
* This takes a TikaCLI commandline and builds the full commandline for
* org.apache.tika.batch.fs.FSBatchProcessCLI.
@@ -55,7 +55,7 @@ class BatchCommandLineBuilder {
translateCommandLine(args, processArgs);
//maybe the user specified a different classpath?!
- if (! jvmOpts.containsKey("-cp") && ! jvmOpts.containsKey("--classpath")) {
+ if (!jvmOpts.containsKey("-cp") && !jvmOpts.containsKey("--classpath")) {
String cp = System.getProperty("java.class.path");
jvmOpts.put("-cp", cp);
}
@@ -68,7 +68,7 @@ class BatchCommandLineBuilder {
}
}
//use the log4j config file inside the app /resources/log4j2_batch_process.properties
- if (! hasLog4j) {
+ if (!hasLog4j) {
jvmOpts.put("-Dlog4j.configuration=log4j_batch_process.properties", "");
}
//now build the full command line
@@ -85,7 +85,7 @@ class BatchCommandLineBuilder {
}
}
//run in headless mode unless the user asks for something else TIKA-2434
- if (! foundHeadlessOption) {
+ if (!foundHeadlessOption) {
fullCommand.add("-Djava.awt.headless=true");
}
fullCommand.add("org.apache.tika.batch.fs.FSBatchProcessCLI");
@@ -115,12 +115,12 @@ class BatchCommandLineBuilder {
/**
* Take the input args and separate them into args that belong on the commandline
* and those that belong as jvm args for the forked process.
- * @param args -- literal args from TikaCLI commandline
+ *
+ * @param args -- literal args from TikaCLI commandline
* @param commandLine args that should be part of the batch commandline
- * @param jvmArgs args that belong as jvm arguments for the forked process
+ * @param jvmArgs args that belong as jvm arguments for the forked process
*/
- private static void mapifyArgs(final String[] args,
- final Map<String, String> commandLine,
+ private static void mapifyArgs(final String[] args, final Map<String, String> commandLine,
final Map<String, String> jvmArgs) {
if (args.length == 0) {
@@ -130,18 +130,18 @@ class BatchCommandLineBuilder {
Matcher matcher = JVM_OPTS_PATTERN.matcher("");
for (int i = 0; i < args.length; i++) {
if (matcher.reset(args[i]).find()) {
- String jvmArg = matcher.group(1)+matcher.group(2);
+ String jvmArg = matcher.group(1) + matcher.group(2);
String v = "";
- if (i < args.length-1 && ! args[i+1].startsWith("-")){
- v = args[i+1];
+ if (i < args.length - 1 && !args[i + 1].startsWith("-")) {
+ v = args[i + 1];
i++;
}
jvmArgs.put(jvmArg, v);
} else if (args[i].startsWith("-")) {
String k = args[i];
String v = "";
- if (i < args.length-1 && ! args[i+1].startsWith("-")){
- v = args[i+1];
+ if (i < args.length - 1 && !args[i + 1].startsWith("-")) {
+ v = args[i + 1];
i++;
}
commandLine.put(k, v);
@@ -149,16 +149,17 @@ class BatchCommandLineBuilder {
}
}
- private static void translateCommandLine(String[] args, Map<String, String> map) throws IOException {
+ private static void translateCommandLine(String[] args, Map<String, String> map)
+ throws IOException {
//if there are only two args and they are both directories, treat the first
//as input and the second as output.
- if (args.length == 2 && !args[0].startsWith("-") && ! args[1].startsWith("-")) {
+ if (args.length == 2 && !args[0].startsWith("-") && !args[1].startsWith("-")) {
Path candInput = Paths.get(args[0]);
Path candOutput = Paths.get(args[1]);
if (Files.isRegularFile(candOutput)) {
- throw new IllegalArgumentException("Can't specify an existing file as the "+
- "second argument for the output directory of a batch process");
+ throw new IllegalArgumentException("Can't specify an existing file as the " +
+ "second argument for the output directory of a batch process");
}
if (Files.isDirectory(candInput)) {
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index 5026a3e..e740e89 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -18,11 +18,6 @@ package org.apache.tika.cli;
import static java.nio.charset.StandardCharsets.UTF_8;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -55,11 +50,21 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
-
+import org.apache.logging.log4j.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.Tika;
import org.apache.tika.batch.BatchProcessDriverCLI;
@@ -108,26 +113,132 @@ import org.apache.tika.sax.WriteOutContentHandler;
import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
import org.apache.tika.xmp.XMPMetadata;
-import org.apache.logging.log4j.Level;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
/**
* Simple command line interface for Apache Tika.
*/
public class TikaCLI {
- private final int MAX_MARK = 20*1024*1024;//20MB
+ private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);
+
+ private final int MAX_MARK = 20 * 1024 * 1024;//20MB
+
+ private final OutputType NO_OUTPUT = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) {
+ return new DefaultHandler();
+ }
+ };
+
+ private final OutputType XML = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ return getTransformerHandler(output, "xml", encoding, prettyPrint);
+ }
+ };
private File extractDir = new File(".");
+ private ParseContext context;
+ private Detector detector;
+ private Parser parser;
+ private TikaConfig config;
+ private String configFilePath;
+ private OutputType type = XML;
+ private boolean recursiveJSON = false;
+ private URI networkURI = null;
+ /**
+ * Output character encoding, or <code>null</code> for platform default
+ */
+ private String encoding = null;
+ private final OutputType TEXT = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ return new BodyContentHandler(getOutputWriter(output, encoding));
+ }
+ };
+ private final OutputType TEXT_MAIN = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ return new BoilerpipeContentHandler(getOutputWriter(output, encoding));
+ }
+ };
+ private final OutputType TEXT_ALL = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ return new WriteOutContentHandler(getOutputWriter(output, encoding));
+ }
+ };
+ private final OutputType METADATA = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
+ return new NoDocumentMetHandler(metadata, writer);
+ }
+ };
+ private final OutputType JSON = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
+ return new NoDocumentJSONMetHandler(metadata, writer);
+ }
+ };
+ private final OutputType XMP = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, final Metadata metadata)
+ throws Exception {
+ final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
+ return new NoDocumentXMPMetaHandler(metadata, writer);
+ }
+ };
+ private final OutputType LANGUAGE = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
+ return new LanguageHandler() {
+ public void endDocument() {
+ writer.println(getLanguage().getLanguage());
+ writer.flush();
+ }
+ };
+ }
+ };
+ private final OutputType DETECT = new OutputType() {
+ @Override
+ public void process(InputStream stream, OutputStream output, Metadata metadata)
+ throws Exception {
+ PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
+ writer.println(detector.detect(stream, metadata).toString());
+ writer.flush();
+ }
+ };
+ /**
+ * Password for opening encrypted documents, or <code>null</code>.
+ */
+ private String password = System.getenv("TIKA_PASSWORD");
+ private DigestingParser.Digester digester = null;
+ private boolean asyncMode = false;
+ private boolean pipeMode = true;
+ private boolean fork = false;
+ private boolean prettyPrint;
- private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);
+ private final OutputType HTML = new OutputType() {
+ @Override
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ return new ExpandedTitleContentHandler(
+ getTransformerHandler(output, "html", encoding, prettyPrint));
+ }
+ };
public TikaCLI() {
context = new ParseContext();
}
+
public static void main(String[] args) throws Exception {
TikaCLI cli = new TikaCLI();
@@ -166,19 +277,8 @@ public class TikaCLI {
}
}
- private boolean testForAsync(String[] args) {
- for (String arg : args) {
- if (arg.equals("-a") || arg.equals("--async")) {
- return true;
- }
- }
- return false;
- }
-
- private static void async(String[] args) throws InterruptedException, PipesException,
- TikaException,
- IOException,
- SAXException {
+ private static void async(String[] args)
+ throws InterruptedException, PipesException, TikaException, IOException, SAXException {
String tikaConfigPath = "";
for (String arg : args) {
if (arg.startsWith("--config=")) {
@@ -201,190 +301,77 @@ public class TikaCLI {
}
}
+ /**
+ * Returns a output writer with the given encoding.
+ *
+ * @param output output stream
+ * @param encoding output encoding,
+ * or <code>null</code> for the platform default
+ * @return output writer
+ * @throws UnsupportedEncodingException if the given encoding is not supported
+ * @see <a href="https://issues.apache.org/jira/browse/TIKA-277">TIKA-277</a>
+ */
+ private static Writer getOutputWriter(OutputStream output, String encoding)
+ throws UnsupportedEncodingException {
+ if (encoding != null) {
+ return new OutputStreamWriter(output, encoding);
+ } else if (System.getProperty("os.name").toLowerCase(Locale.ROOT).startsWith("mac os x")) {
+ // TIKA-324: Override the default encoding on Mac OS X
+ return new OutputStreamWriter(output, UTF_8);
+ } else {
+ return new OutputStreamWriter(output, Charset.defaultCharset());
+ }
+ }
+
+ /**
+ * Returns a transformer handler that serializes incoming SAX events
+ * to XHTML or HTML (depending the given method) using the given output
+ * encoding.
+ *
+ * @param output output stream
+ * @param method "xml" or "html"
+ * @param encoding output encoding,
+ * or <code>null</code> for the platform default
+ * @return {@link System#out} transformer handler
+ * @throws TransformerConfigurationException if the transformer can not be created
+ * @see <a href="https://issues.apache.org/jira/browse/TIKA-277">TIKA-277</a>
+ */
+ private static TransformerHandler getTransformerHandler(OutputStream output, String method,
+ String encoding, boolean prettyPrint)
+ throws TransformerConfigurationException {
+ SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
+ TransformerHandler handler = factory.newTransformerHandler();
+ handler.getTransformer().setOutputProperty(OutputKeys.METHOD, method);
+ handler.getTransformer().setOutputProperty(OutputKeys.INDENT, prettyPrint ? "yes" : "no");
+ if (encoding != null) {
+ handler.getTransformer().setOutputProperty(OutputKeys.ENCODING, encoding);
+ }
+ handler.setResult(new StreamResult(output));
+ return handler;
+ }
+
+ private boolean testForAsync(String[] args) {
+ for (String arg : args) {
+ if (arg.equals("-a") || arg.equals("--async")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void extractInlineImagesFromPDFs() {
if (configFilePath == null && context.get(PDFParserConfig.class) == null) {
PDFParserConfig pdfParserConfig = new PDFParserConfig();
pdfParserConfig.setExtractInlineImages(true);
String warn = "As a convenience, TikaCLI has turned on extraction of\n" +
"inline images for the PDFParser (TIKA-2374).\n" +
- "Aside from the -z option, this is not the default behavior\n"+
+ "Aside from the -z option, this is not the default behavior\n" +
"in Tika generally or in tika-server.";
LOG.info(warn);
context.set(PDFParserConfig.class, pdfParserConfig);
}
}
- private class OutputType {
- public void process(
- InputStream input, OutputStream output, Metadata metadata)
- throws Exception {
- Parser p = parser;
- if (fork) {
- p = new ForkParser(TikaCLI.class.getClassLoader(), p);
- }
- ContentHandler handler = getContentHandler(output, metadata);
-
- p.parse(input, handler, metadata, context);
- // fix for TIKA-596: if a parser doesn't generate
- // XHTML output, the lack of an output document prevents
- // metadata from being output: this fixes that
- if (handler instanceof NoDocumentMetHandler){
- NoDocumentMetHandler metHandler = (NoDocumentMetHandler)handler;
- if(!metHandler.metOutput()){
- metHandler.endDocument();
- }
- }
- }
-
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- throw new UnsupportedOperationException();
- }
-
- }
-
- private final OutputType XML = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- return getTransformerHandler(output, "xml", encoding, prettyPrint);
- }
- };
-
- private final OutputType HTML = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- return new ExpandedTitleContentHandler(getTransformerHandler(output, "html", encoding, prettyPrint));
- }
- };
-
- private final OutputType TEXT = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- return new BodyContentHandler(getOutputWriter(output, encoding));
- }
- };
-
- private final OutputType NO_OUTPUT = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) {
- return new DefaultHandler();
- }
- };
-
- private final OutputType TEXT_MAIN = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- return new BoilerpipeContentHandler(getOutputWriter(output, encoding));
- }
- };
-
- private final OutputType TEXT_ALL = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- return new WriteOutContentHandler(getOutputWriter(output, encoding));
- }
- };
-
- private final OutputType METADATA = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- final PrintWriter writer =
- new PrintWriter(getOutputWriter(output, encoding));
- return new NoDocumentMetHandler(metadata, writer);
- }
- };
-
- private final OutputType JSON = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- final PrintWriter writer =
- new PrintWriter(getOutputWriter(output, encoding));
- return new NoDocumentJSONMetHandler(metadata, writer);
- }
- };
-
- private final OutputType XMP = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, final Metadata metadata) throws Exception {
- final PrintWriter writer =
- new PrintWriter(getOutputWriter(output, encoding));
- return new NoDocumentXMPMetaHandler(metadata, writer);
- }
- };
-
- private final OutputType LANGUAGE = new OutputType() {
- @Override
- protected ContentHandler getContentHandler(
- OutputStream output, Metadata metadata) throws Exception {
- final PrintWriter writer =
- new PrintWriter(getOutputWriter(output, encoding));
- return new LanguageHandler() {
- public void endDocument() {
- writer.println(getLanguage().getLanguage());
- writer.flush();
- }
- };
- }
- };
-
- private final OutputType DETECT = new OutputType() {
- @Override
- public void process(
- InputStream stream, OutputStream output, Metadata metadata)
- throws Exception {
- PrintWriter writer =
- new PrintWriter(getOutputWriter(output, encoding));
- writer.println(detector.detect(stream, metadata).toString());
- writer.flush();
- }
- };
-
-
- private ParseContext context;
-
- private Detector detector;
-
- private Parser parser;
-
- private TikaConfig config;
-
- private String configFilePath;
-
- private OutputType type = XML;
-
- private boolean recursiveJSON = false;
-
- private URI networkURI = null;
-
- /**
- * Output character encoding, or <code>null</code> for platform default
- */
- private String encoding = null;
-
- /**
- * Password for opening encrypted documents, or <code>null</code>.
- */
- private String password = System.getenv("TIKA_PASSWORD");
-
- private DigestingParser.Digester digester = null;
-
- private boolean asyncMode = false;
-
- private boolean pipeMode = true;
-
- private boolean fork = false;
-
- private boolean prettyPrint;
-
public void process(String arg) throws Exception {
if (arg.equals("-?") || arg.equals("--help")) {
pipeMode = false;
@@ -396,7 +383,7 @@ public class TikaCLI {
org.apache.logging.log4j.core.config.Configurator.setRootLevel(Level.DEBUG);
} else if (arg.equals("-g") || arg.equals("--gui")) {
pipeMode = false;
- if (configFilePath != null){
+ if (configFilePath != null) {
TikaGUI.main(new String[]{configFilePath});
} else {
TikaGUI.main(new String[0]);
@@ -405,18 +392,19 @@ public class TikaCLI {
pipeMode = false;
displayParsers(false, false);
} else if (arg.equals("--list-detector") || arg.equals("--list-detectors")) {
- pipeMode = false;
- displayDetectors();
+ pipeMode = false;
+ displayDetectors();
} else if (arg.equals("--list-parser-detail") || arg.equals("--list-parser-details")) {
pipeMode = false;
displayParsers(true, false);
- } else if (arg.equals("--list-parser-detail-apt") || arg.equals("--list-parser-details-apt")) {
+ } else if (arg.equals("--list-parser-detail-apt") ||
+ arg.equals("--list-parser-details-apt")) {
pipeMode = false;
displayParsers(true, true);
- } else if(arg.equals("--list-met-models")){
+ } else if (arg.equals("--list-met-models")) {
pipeMode = false;
displayMetModels();
- } else if(arg.equals("--list-supported-types")){
+ } else if (arg.equals("--list-supported-types")) {
pipeMode = false;
displaySupportedTypes();
} else if (arg.startsWith("--compare-file-magic=")) {
@@ -434,18 +422,16 @@ public class TikaCLI {
} else if (arg.equals("--dump-static-full-config")) {
pipeMode = false;
dumpConfig(TikaConfigSerializer.Mode.STATIC_FULL);
- } else if (arg.equals("--container-aware")
- || arg.equals("--container-aware-detector")) {
+ } else if (arg.equals("--container-aware") || arg.equals("--container-aware-detector")) {
// ignore, as container-aware detectors are now always used
} else if (arg.equals("-f") || arg.equals("--fork")) {
fork = true;
} else if (arg.equals("-a") || arg.equals("--async")) {
asyncMode = true;
- } else if (arg.startsWith("--config=")) {
+ } else if (arg.startsWith("--config=")) {
configFilePath = arg.substring("--config=".length());
} else if (arg.startsWith("--digest=")) {
- digester = new CommonsDigester(MAX_MARK,
- arg.substring("--digest=".length()));
+ digester = new CommonsDigester(MAX_MARK, arg.substring("--digest=".length()));
} else if (arg.startsWith("-e")) {
encoding = arg.substring("-e".length());
} else if (arg.startsWith("--encoding=")) {
@@ -454,7 +440,7 @@ public class TikaCLI {
password = arg.substring("-p".length());
} else if (arg.startsWith("--password=")) {
password = arg.substring("--password=".length());
- } else if (arg.equals("-j") || arg.equals("--json")) {
+ } else if (arg.equals("-j") || arg.equals("--json")) {
type = JSON;
} else if (arg.equals("-J") || arg.equals("--jsonRecursive")) {
recursiveJSON = true;
@@ -490,11 +476,11 @@ public class TikaCLI {
context.set(EmbeddedDocumentExtractor.class, new FileEmbeddedDocumentExtractor());
} else if (arg.equals("-r") || arg.equals("--pretty-print")) {
prettyPrint = true;
- } else if (arg.equals("-p") || arg.equals("--port")
- || arg.equals("-s") || arg.equals("--server")) {
+ } else if (arg.equals("-p") || arg.equals("--port") || arg.equals("-s") ||
+ arg.equals("--server")) {
throw new IllegalArgumentException(
- "As of Tika 2.0, the server option is no longer supported in tika-app.\n"+
- "See https://wiki.apache.org/tika/TikaJAXRS for usage.");
+ "As of Tika 2.0, the server option is no longer supported in tika-app.\n" +
+ "See https://wiki.apache.org/tika/TikaJAXRS for usage.");
} else if (arg.startsWith("-c")) {
networkURI = new URI(arg.substring("-c".length()));
} else if (arg.startsWith("--client=")) {
@@ -502,10 +488,10 @@ public class TikaCLI {
} else {
pipeMode = false;
configure();
-
+
if (arg.equals("-")) {
- try (InputStream stream = TikaInputStream.get(
- new CloseShieldInputStream(System.in))) {
+ try (InputStream stream = TikaInputStream
+ .get(new CloseShieldInputStream(System.in))) {
type.process(stream, System.out, new Metadata());
}
} else {
@@ -520,8 +506,7 @@ public class TikaCLI {
handleRecursiveJson(url, System.out);
} else {
Metadata metadata = new Metadata();
- try (InputStream input =
- TikaInputStream.get(url, metadata)) {
+ try (InputStream input = TikaInputStream.get(url, metadata)) {
type.process(input, System.out, metadata);
} finally {
System.out.flush();
@@ -535,27 +520,29 @@ public class TikaCLI {
configure();
TikaConfig localConfig = (config == null) ? TikaConfig.getDefaultConfig() : config;
- TikaConfigSerializer.serialize(localConfig, mode,
- new OutputStreamWriter(System.out, UTF_8), UTF_8);
+ TikaConfigSerializer
+ .serialize(localConfig, mode, new OutputStreamWriter(System.out, UTF_8), UTF_8);
}
- private void handleRecursiveJson(URL url, OutputStream output) throws IOException, SAXException, TikaException {
+ private void handleRecursiveJson(URL url, OutputStream output)
+ throws IOException, SAXException, TikaException {
Metadata metadata = new Metadata();
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
RecursiveParserWrapperHandler handler =
- new RecursiveParserWrapperHandler(getContentHandlerFactory(type),
- -1, config.getMetadataFilter());
+ new RecursiveParserWrapperHandler(getContentHandlerFactory(type), -1,
+ config.getMetadataFilter());
try (InputStream input = TikaInputStream.get(url, metadata)) {
wrapper.parse(input, handler, metadata, context);
}
JsonMetadataList.setPrettyPrinting(prettyPrint);
- try(Writer writer = getOutputWriter(output, encoding)) {
+ try (Writer writer = getOutputWriter(output, encoding)) {
JsonMetadataList.toJson(handler.getMetadataList(), writer);
}
}
private ContentHandlerFactory getContentHandlerFactory(OutputType type) {
- BasicContentHandlerFactory.HANDLER_TYPE handlerType = BasicContentHandlerFactory.HANDLER_TYPE.IGNORE;
+ BasicContentHandlerFactory.HANDLER_TYPE handlerType =
+ BasicContentHandlerFactory.HANDLER_TYPE.IGNORE;
if (type.equals(HTML)) {
handlerType = BasicContentHandlerFactory.HANDLER_TYPE.HTML;
} else if (type.equals(XML)) {
@@ -569,6 +556,7 @@ public class TikaCLI {
}
return new BasicContentHandlerFactory(handlerType, -1);
}
+
private void usage() {
PrintStream out = System.out;
out.println("usage: java -jar tika-app.jar [option...] [file...]");
@@ -582,7 +570,8 @@ public class TikaCLI {
out.println(" -f or --fork Use Fork Mode for out-of-process extraction");
out.println();
out.println(" --config=<tika-config.xml>");
- out.println(" TikaConfig file. Must be specified before -g, -s, -f or the dump-x-config !");
+ out.println(
+ " TikaConfig file. Must be specified before -g, -s, -f or the dump-x-config !");
out.println(" --dump-minimal-config Print minimal TikaConfig");
out.println(" --dump-current-config Print current TikaConfig");
out.println(" --dump-static-config Print static config");
@@ -591,7 +580,8 @@ public class TikaCLI {
out.println(" -x or --xml Output XHTML content (default)");
out.println(" -h or --html Output HTML content");
out.println(" -t or --text Output plain text content (body)");
- out.println(" -T or --text-main Output plain text content (main content only via boilerpipe handler)");
+ out.println(
+ " -T or --text-main Output plain text content (main content only via boilerpipe handler)");
out.println(" -A or --text-all Output all text content");
out.println(" -m or --metadata Output only metadata");
out.println(" -j or --json Output metadata in JSON");
@@ -609,7 +599,8 @@ public class TikaCLI {
out.println(" -pX or --password=X Use document password X");
out.println(" -z or --extract Extract all attachements into current directory");
out.println(" --extract-dir=<dir> Specify target directory for -z");
- out.println(" -r or --pretty-print For JSON, XML and XHTML outputs, adds newlines and");
+ out.println(
+ " -r or --pretty-print For JSON, XML and XHTML outputs, adds newlines and");
out.println(" whitespace, for better readability");
out.println();
out.println(" --list-parsers");
@@ -617,7 +608,8 @@ public class TikaCLI {
out.println(" --list-parser-details");
out.println(" List the available document parsers and their supported mime types");
out.println(" --list-parser-details-apt");
- out.println(" List the available document parsers and their supported mime types in apt format.");
+ out.println(
+ " List the available document parsers and their supported mime types in apt format.");
out.println(" --list-detectors");
out.println(" List the available document detectors");
out.println(" --list-met-models");
@@ -627,7 +619,8 @@ public class TikaCLI {
out.println();
out.println();
out.println(" --compare-file-magic=<dir>");
- out.println(" Compares Tika's known media types to the File(1) tool's magic directory");
+ out.println(
+ " Compares Tika's known media types to the File(1) tool's magic directory");
out.println("Description:");
out.println(" Apache Tika will parse the file(s) specified on the");
out.println(" command line and output the extracted text content");
@@ -660,15 +653,18 @@ public class TikaCLI {
out.println(" -numConsumers Number of processing threads");
out.println(" -bc Batch config file");
out.println(" -maxRestarts Maximum number of times the ");
- out.println(" watchdog process will restart the forked process.");
+ out.println(
+ " watchdog process will restart the forked process.");
out.println(" -timeoutThresholdMillis Number of milliseconds allowed to a parse");
- out.println(" before the process is terminated and restarted");
+ out.println(
+ " before the process is terminated and restarted");
out.println(" -fileList List of files to process, with");
out.println(" paths relative to the input directory");
out.println(" -includeFilePat Regular expression to determine which");
out.println(" files to process, e.g. \"(?i)\\.pdf\"");
out.println(" -excludeFilePat Regular expression to determine which");
- out.println(" files to avoid processing, e.g. \"(?i)\\.pdf\"");
+ out.println(
+ " files to avoid processing, e.g. \"(?i)\\.pdf\"");
out.println(" -maxFileSizeBytes Skip files longer than this value");
out.println();
out.println(" Control the type of output with -x, -h, -t and/or -J.");
@@ -691,12 +687,10 @@ public class TikaCLI {
}
private boolean testForBatch(String[] args) {
- if (args.length == 2 && ! args[0].startsWith("-")
- && ! args[1].startsWith("-")) {
+ if (args.length == 2 && !args[0].startsWith("-") && !args[1].startsWith("-")) {
Path inputCand = Paths.get(args[0]);
Path outputCand = Paths.get(args[1]);
- if (Files.isDirectory(inputCand) &&
- !Files.isRegularFile(outputCand)) {
+ if (Files.isDirectory(inputCand) && !Files.isRegularFile(outputCand)) {
return true;
}
}
@@ -709,7 +703,6 @@ public class TikaCLI {
return false;
}
-
private void configure() throws TikaException, IOException, SAXException {
if (networkURI != null) {
@@ -732,19 +725,19 @@ public class TikaCLI {
context.set(PasswordProvider.class, new SimplePasswordProvider(password));
}
- private void displayMetModels(){
+ private void displayMetModels() {
Class<?>[] modelClasses = Metadata.class.getInterfaces();
Arrays.sort(modelClasses, Comparator.comparing(Class::getName));
- for (Class<?> modelClass: modelClasses) {
+ for (Class<?> modelClass : modelClasses) {
// we don't care about internal Tika met classes
// if we do, then we can take this conditional out
if (!modelClass.getSimpleName().contains("Tika")) {
System.out.println(modelClass.getSimpleName());
Field[] keyFields = modelClass.getFields();
Arrays.sort(keyFields, Comparator.comparing(Field::getName));
- for (Field keyField: keyFields) {
- System.out.println(" "+keyField.getName());
+ for (Field keyField : keyFields) {
+ System.out.println(" " + keyField.getName());
}
}
}
@@ -755,42 +748,52 @@ public class TikaCLI {
* If a parser is a composite parser, it will list the
* sub parsers and their mime-types.
*/
- private void displayParsers(boolean includeMimeTypes, boolean aptListFormat) throws TikaException, IOException, SAXException {
+ private void displayParsers(boolean includeMimeTypes, boolean aptListFormat)
+ throws TikaException, IOException, SAXException {
configure();
displayParser(parser, includeMimeTypes, aptListFormat, 3);
}
-
+
private void displayParser(Parser p, boolean includeMimeTypes, boolean apt, int i) {
String decorated = null;
if (p instanceof ParserDecorator) {
- ParserDecorator pd = (ParserDecorator)p;
+ ParserDecorator pd = (ParserDecorator) p;
decorated = " (Wrapped by " + pd.getDecorationName() + ")";
p = pd.getWrappedParser();
}
-
+
boolean isComposite = (p instanceof CompositeParser);
String name = p.getClass().getName();
-
+
if (apt) {
- name = name.substring(0, name.lastIndexOf(".") + 1) + "{{{./api/" + name.replace(".", "/") + "}" + name.substring(name.lastIndexOf(".") + 1) + "}}";
+ name = name.substring(0, name.lastIndexOf(".") + 1) + "{{{./api/" +
+ name.replace(".", "/") + "}" + name.substring(name.lastIndexOf(".") + 1) + "}}";
} else if (decorated != null) {
name += decorated;
}
- if ((apt && !isComposite) || !apt) { // Don't display Composite parsers in the apt output.
- System.out.println(indent(i) + ((apt) ? "* " : "") + name + (isComposite ? " (Composite Parser):" : ""));
- if (apt) System.out.println();
+ if ((apt && !isComposite) ||
+ !apt) { // Don't display Composite parsers in the apt output.
+ System.out.println(indent(i) + ((apt) ? "* " : "") + name +
+ (isComposite ? " (Composite Parser):" : ""));
+ if (apt) {
+ System.out.println();
+ }
if (includeMimeTypes && !isComposite) {
for (MediaType mt : p.getSupportedTypes(context)) {
System.out.println(indent(i + 3) + ((apt) ? "* " : "") + mt);
- if (apt) System.out.println();
+ if (apt) {
+ System.out.println();
+ }
}
}
}
-
+
if (isComposite) {
- Parser[] subParsers = sortParsers(invertMediaTypeMap(((CompositeParser) p).getParsers()));
- for(Parser sp : subParsers) {
- displayParser(sp, includeMimeTypes, apt, i + ((apt) ? 0 : 3)); // Don't indent for Composites in apt.
+ Parser[] subParsers =
+ sortParsers(invertMediaTypeMap(((CompositeParser) p).getParsers()));
+ for (Parser sp : subParsers) {
+ displayParser(sp, includeMimeTypes, apt,
+ i + ((apt) ? 0 : 3)); // Don't indent for Composites in apt.
}
}
}
@@ -804,15 +807,15 @@ public class TikaCLI {
configure();
displayDetector(detector, 0);
}
-
+
private void displayDetector(Detector d, int i) {
boolean isComposite = (d instanceof CompositeDetector);
String name = d.getClass().getName();
System.out.println(indent(i) + name + (isComposite ? " (Composite Detector):" : ""));
if (isComposite) {
- List<Detector> subDetectors = ((CompositeDetector)d).getDetectors();
- for(Detector sd : subDetectors) {
- displayDetector(sd, i+2);
+ List<Detector> subDetectors = ((CompositeDetector) d).getDetectors();
+ for (Detector sd : subDetectors) {
+ displayDetector(sd, i + 2);
}
}
}
@@ -833,8 +836,8 @@ public class TikaCLI {
}
private Map<Parser, Set<MediaType>> invertMediaTypeMap(Map<MediaType, Parser> supported) {
- Map<Parser,Set<MediaType>> parsers = new HashMap<>();
- for(Entry<MediaType, Parser> e : supported.entrySet()) {
+ Map<Parser, Set<MediaType>> parsers = new HashMap<>();
+ for (Entry<MediaType, Parser> e : supported.entrySet()) {
if (!parsers.containsKey(e.getValue())) {
parsers.put(e.getValue(), new HashSet<>());
}
@@ -863,44 +866,43 @@ public class TikaCLI {
Parser p = parsers.get(type);
if (p != null) {
if (p instanceof CompositeParser) {
- p = ((CompositeParser)p).getParsers().get(type);
+ p = ((CompositeParser) p).getParsers().get(type);
}
System.out.println(" parser: " + p.getClass().getName());
}
}
}
-
+
/**
- * Compares our mime types registry with the File(1) tool's
- * directory of (uncompiled) Magic entries.
+ * Compares our mime types registry with the File(1) tool's
+ * directory of (uncompiled) Magic entries.
* (Well, those with mimetypes anyway)
+ *
* @param magicDir Path to the magic directory
*/
private void compareFileMagic(String magicDir) throws Exception {
Set<String> tikaLacking = new TreeSet<>();
Set<String> tikaNoMagic = new TreeSet<>();
-
+
// Sanity check
File dir = new File(magicDir);
- if ((new File(dir, "elf")).exists() &&
- (new File(dir, "mime")).exists() &&
- (new File(dir, "vorbis")).exists()) {
+ if ((new File(dir, "elf")).exists() && (new File(dir, "mime")).exists() &&
+ (new File(dir, "vorbis")).exists()) {
// Looks plausible
} else {
throw new IllegalArgumentException(
- magicDir + " doesn't seem to hold uncompressed file magic entries");
+ magicDir + " doesn't seem to hold uncompressed file magic entries");
}
-
+
// Find all the mimetypes in the directory
Set<String> fileMimes = new HashSet<>();
for (File mf : dir.listFiles()) {
if (mf.isFile()) {
- BufferedReader r = new BufferedReader(new InputStreamReader(
- new FileInputStream(mf), UTF_8));
+ BufferedReader r =
+ new BufferedReader(new InputStreamReader(new FileInputStream(mf), UTF_8));
String line;
while ((line = r.readLine()) != null) {
- if (line.startsWith("!:mime") ||
- line.startsWith("#!:mime")) {
+ if (line.startsWith("!:mime") || line.startsWith("#!:mime")) {
String mime = line.substring(7).trim();
fileMimes.add(mime);
}
@@ -908,7 +910,7 @@ public class TikaCLI {
r.close();
}
}
-
+
// See how those compare to the Tika ones
TikaConfig config = TikaConfig.getDefaultConfig();
MimeTypes mimeTypes = config.getMimeRepository();
@@ -916,16 +918,16 @@ public class TikaCLI {
for (String mime : fileMimes) {
try {
final MimeType type = mimeTypes.getRegisteredMimeType(mime);
-
+
if (type == null) {
// Tika doesn't know about this one
tikaLacking.add(mime);
} else {
// Tika knows about this one!
-
+
// Does Tika have magic for it?
boolean hasMagic = type.hasMagic();
-
+
// How about the children?
if (!hasMagic) {
for (MediaType child : registry.getChildTypes(type.getType())) {
@@ -935,7 +937,7 @@ public class TikaCLI {
}
}
}
-
+
// How about the parents?
MimeType parentType = type;
while (parentType != null && !hasMagic) {
@@ -946,8 +948,8 @@ public class TikaCLI {
// Check the parent next
MediaType parent = registry.getSupertype(type.getType());
if (parent == MediaType.APPLICATION_XML ||
- parent == MediaType.TEXT_PLAIN ||
- parent == MediaType.OCTET_STREAM) {
+ parent == MediaType.TEXT_PLAIN ||
+ parent == MediaType.OCTET_STREAM) {
// Stop checking parents if we hit a top level type
parent = null;
}
@@ -967,7 +969,7 @@ public class TikaCLI {
// Silently skip
}
}
-
+
// Check how many tika knows about
int tikaTypes = 0;
int tikaAliases = 0;
@@ -975,91 +977,147 @@ public class TikaCLI {
tikaTypes++;
tikaAliases += registry.getAliases(type).size();
}
-
+
// Report
System.out.println("Tika knows about " + tikaTypes + " unique mime types");
- System.out.println("Tika knows about " + (tikaTypes+tikaAliases) + " mime types including aliases");
- System.out.println("The File Magic directory knows about " + fileMimes.size() + " unique mime types");
+ System.out.println(
+ "Tika knows about " + (tikaTypes + tikaAliases) + " mime types including aliases");
+ System.out.println(
+ "The File Magic directory knows about " + fileMimes.size() + " unique mime types");
System.out.println();
System.out.println("The following mime types are known to File but not Tika:");
for (String mime : tikaLacking) {
System.out.println(" " + mime);
}
System.out.println();
- System.out.println("The following mime types from File have no Tika magic (but their children might):");
+ System.out.println(
+ "The following mime types from File have no Tika magic (but their children might):");
for (String mime : tikaNoMagic) {
System.out.println(" " + mime);
}
}
- /**
- * Returns a output writer with the given encoding.
- *
- * @see <a href="https://issues.apache.org/jira/browse/TIKA-277">TIKA-277</a>
- * @param output output stream
- * @param encoding output encoding,
- * or <code>null</code> for the platform default
- * @return output writer
- * @throws UnsupportedEncodingException
- * if the given encoding is not supported
- */
- private static Writer getOutputWriter(OutputStream output, String encoding)
- throws UnsupportedEncodingException {
- if (encoding != null) {
- return new OutputStreamWriter(output, encoding);
- } else if (System.getProperty("os.name")
- .toLowerCase(Locale.ROOT).startsWith("mac os x")) {
- // TIKA-324: Override the default encoding on Mac OS X
- return new OutputStreamWriter(output, UTF_8);
- } else {
- return new OutputStreamWriter(output, Charset.defaultCharset());
+ private static class NoDocumentMetHandler extends DefaultHandler {
+
+ protected final Metadata metadata;
+
+ protected PrintWriter writer;
+
+ private boolean metOutput;
+
+ public NoDocumentMetHandler(Metadata metadata, PrintWriter writer) {
+ this.metadata = metadata;
+ this.writer = writer;
+ this.metOutput = false;
}
+
+ @Override
+ public void endDocument() {
+ String[] names = metadata.names();
+ Arrays.sort(names);
+ outputMetadata(names);
+ writer.flush();
+ this.metOutput = true;
+ }
+
+ public void outputMetadata(String[] names) {
+ for (String name : names) {
+ for (String value : metadata.getValues(name)) {
+ writer.println(name + ": " + value);
+ }
+ }
+ }
+
+ public boolean metOutput() {
+ return this.metOutput;
+ }
+
}
/**
- * Returns a transformer handler that serializes incoming SAX events
- * to XHTML or HTML (depending the given method) using the given output
- * encoding.
- *
- * @see <a href="https://issues.apache.org/jira/browse/TIKA-277">TIKA-277</a>
- * @param output output stream
- * @param method "xml" or "html"
- * @param encoding output encoding,
- * or <code>null</code> for the platform default
- * @return {@link System#out} transformer handler
- * @throws TransformerConfigurationException
- * if the transformer can not be created
+ * Outputs the Tika metadata as XMP using the Tika XMP module
*/
- private static TransformerHandler getTransformerHandler(
- OutputStream output, String method, String encoding, boolean prettyPrint)
- throws TransformerConfigurationException {
- SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
- TransformerHandler handler = factory.newTransformerHandler();
- handler.getTransformer().setOutputProperty(OutputKeys.METHOD, method);
- handler.getTransformer().setOutputProperty(OutputKeys.INDENT, prettyPrint ? "yes" : "no");
- if (encoding != null) {
- handler.getTransformer().setOutputProperty(
- OutputKeys.ENCODING, encoding);
+ private static class NoDocumentXMPMetaHandler extends DefaultHandler {
+ protected final Metadata metadata;
+
+ protected PrintWriter writer;
+
+ public NoDocumentXMPMetaHandler(Metadata metadata, PrintWriter writer) {
+ this.metadata = metadata;
+ this.writer = writer;
+ }
+
+ @Override
+ public void endDocument() throws SAXException {
+ try {
+ XMPMetadata xmp = new XMPMetadata(metadata);
+ String result;
+ result = xmp.toString();
+ writer.write(result);
+ writer.flush();
+ } catch (TikaException e) {
+ throw new SAXException(e);
+ }
}
- handler.setResult(new StreamResult(output));
- return handler;
}
- private class FileEmbeddedDocumentExtractor
- implements EmbeddedDocumentExtractor {
+ private static class SimplePasswordProvider implements PasswordProvider, Serializable {
+ private final String password;
+
+ public SimplePasswordProvider(String password) {
+ this.password = password;
+ }
+
+ @Override
+ public String getPassword(Metadata metadata) {
+ return password;
+ }
+ }
+
+ private class OutputType {
+ public void process(InputStream input, OutputStream output, Metadata metadata)
+ throws Exception {
+ Parser p = parser;
+ if (fork) {
+ p = new ForkParser(TikaCLI.class.getClassLoader(), p);
+ }
+ ContentHandler handler = getContentHandler(output, metadata);
+
+ p.parse(input, handler, metadata, context);
+ // fix for TIKA-596: if a parser doesn't generate
+ // XHTML output, the lack of an output document prevents
+ // metadata from being output: this fixes that
+ if (handler instanceof NoDocumentMetHandler) {
+ NoDocumentMetHandler metHandler = (NoDocumentMetHandler) handler;
+ if (!metHandler.metOutput()) {
+ metHandler.endDocument();
+ }
+ }
+ }
+
+ protected ContentHandler getContentHandler(OutputStream output, Metadata metadata)
+ throws Exception {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private class FileEmbeddedDocumentExtractor implements EmbeddedDocumentExtractor {
- private int count = 0;
private final TikaConfig config = TikaConfig.getDefaultConfig();
- private final EmbeddedStreamTranslator embeddedStreamTranslator = new DefaultEmbeddedStreamTranslator();
+ private final EmbeddedStreamTranslator embeddedStreamTranslator =
+ new DefaultEmbeddedStreamTranslator();
+ private int count = 0;
public boolean shouldParseEmbedded(Metadata metadata) {
return true;
}
- public void parseEmbedded(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, boolean outputHtml) throws SAXException, IOException {
+ public void parseEmbedded(InputStream inputStream, ContentHandler contentHandler,
+ Metadata metadata, boolean outputHtml)
+ throws SAXException, IOException {
- if (! inputStream.markSupported()) {
+ if (!inputStream.markSupported()) {
inputStream = TikaInputStream.get(inputStream);
}
MediaType contentType = detector.detect(inputStream, metadata);
@@ -1078,11 +1136,12 @@ public class TikaCLI {
throw new IOException("unable to create directory \"" + parent + "\"");
}
}
- System.out.println("Extracting '"+name+"' ("+contentType+") to " + outputFile);
+ System.out.println("Extracting '" + name + "' (" + contentType + ") to " + outputFile);
try (FileOutputStream os = new FileOutputStream(outputFile)) {
if (embeddedStreamTranslator.shouldTranslate(inputStream, metadata)) {
- try (InputStream translated = embeddedStreamTranslator.translate(inputStream, metadata)) {
+ try (InputStream translated = embeddedStreamTranslator
+ .translate(inputStream, metadata)) {
IOUtils.copy(translated, os);
}
} else {
@@ -1092,19 +1151,16 @@ public class TikaCLI {
//
// being a CLI program messages should go to the stderr too
//
- String msg = String.format(
- Locale.ROOT,
- "Ignoring unexpected exception trying to save embedded file %s (%s)",
- name,
- e.getMessage()
- );
+ String msg = String.format(Locale.ROOT,
+ "Ignoring unexpected exception trying to save embedded file %s (%s)", name,
+ e.getMessage());
LOG.warn(msg, e);
}
}
private File getOutputFile(String name, Metadata metadata, MediaType contentType) {
String ext = getExtension(contentType);
- if (name.indexOf('.')==-1 && contentType!=null) {
+ if (name.indexOf('.') == -1 && contentType != null) {
name += ext;
}
@@ -1122,7 +1178,7 @@ public class TikaCLI {
}
if (normalizedName == null) {
- normalizedName = "file"+count++ +ext;
+ normalizedName = "file" + count++ + ext;
}
//strip off initial C:/ or ~/ or /
int prefixLength = FilenameUtils.getPrefixLength(normalizedName);
@@ -1133,15 +1189,15 @@ public class TikaCLI {
//if file already exists, prepend uuid
if (outputFile.exists()) {
String fileName = FilenameUtils.getName(normalizedName);
- outputFile = new File(extractDir, UUID.randomUUID().toString()+"-"+fileName);
+ outputFile = new File(extractDir, UUID.randomUUID().toString() + "-" + fileName);
}
return outputFile;
}
private String getExtension(MediaType contentType) {
try {
- String ext = config.getMimeRepository().forName(
- contentType.toString()).getExtension();
+ String ext =
+ config.getMimeRepository().forName(contentType.toString()).getExtension();
if (ext == null) {
return ".bin";
} else {
@@ -1155,86 +1211,17 @@ public class TikaCLI {
}
}
- private static class NoDocumentMetHandler extends DefaultHandler {
-
- protected final Metadata metadata;
-
- protected PrintWriter writer;
-
- private boolean metOutput;
-
- public NoDocumentMetHandler(Metadata metadata, PrintWriter writer){
- this.metadata = metadata;
- this.writer = writer;
- this.metOutput = false;
- }
-
- @Override
- public void endDocument() {
- String[] names = metadata.names();
- Arrays.sort(names);
- outputMetadata(names);
- writer.flush();
- this.metOutput = true;
- }
-
- public void outputMetadata(String[] names) {
- for (String name : names) {
- for(String value : metadata.getValues(name)) {
- writer.println(name + ": " + value);
- }
- }
- }
-
- public boolean metOutput(){
- return this.metOutput;
- }
-
- }
-
- /**
- * Outputs the Tika metadata as XMP using the Tika XMP module
- */
- private static class NoDocumentXMPMetaHandler extends DefaultHandler
- {
- protected final Metadata metadata;
-
- protected PrintWriter writer;
-
- public NoDocumentXMPMetaHandler(Metadata metadata, PrintWriter writer){
- this.metadata = metadata;
- this.writer = writer;
- }
-
- @Override
- public void endDocument() throws SAXException
- {
- try
- {
- XMPMetadata xmp = new XMPMetadata(metadata);
- String result;
- result = xmp.toString();
- writer.write(result);
- writer.flush();
- }
- catch (TikaException e)
- {
- throw new SAXException(e);
- }
- }
- }
-
private class NoDocumentJSONMetHandler extends DefaultHandler {
protected final Metadata metadata;
-
+
protected PrintWriter writer;
public NoDocumentJSONMetHandler(Metadata metadata, PrintWriter writer) {
this.metadata = metadata;
this.writer = writer;
}
-
+
@Override
public void endDocument() throws SAXException {
try {
@@ -1244,20 +1231,6 @@ public class TikaCLI {
} catch (IOException e) {
throw new SAXException(e);
}
- }
- }
-
-
- private static class SimplePasswordProvider
- implements PasswordProvider, Serializable {
- private final String password;
- public SimplePasswordProvider(String password) {
- this.password = password;
- }
-
- @Override
- public String getPassword(Metadata metadata) {
- return password;
}
}
}
diff --git a/tika-app/src/main/java/org/apache/tika/gui/ParsingTransferHandler.java b/tika-app/src/main/java/org/apache/tika/gui/ParsingTransferHandler.java
index 7d9eb07..721ae13 100644
--- a/tika-app/src/main/java/org/apache/tika/gui/ParsingTransferHandler.java
+++ b/tika-app/src/main/java/org/apache/tika/gui/ParsingTransferHandler.java
@@ -26,7 +26,6 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
-
import javax.swing.Icon;
import javax.swing.JComponent;
import javax.swing.TransferHandler;
@@ -40,29 +39,47 @@ class ParsingTransferHandler extends TransferHandler {
* Serial version UID.
*/
private static final long serialVersionUID = -557932290014044494L;
-
- private final TransferHandler delegate;
-
- private final TikaGUI tika;
-
private static DataFlavor uriListFlavor;
private static DataFlavor urlListFlavor;
+
static {
- try {
- uriListFlavor = new DataFlavor("text/uri-list;class=java.lang.String");
- urlListFlavor = new DataFlavor("text/plain;class=java.lang.String");
- } catch (ClassNotFoundException e) {
- }
+ try {
+ uriListFlavor = new DataFlavor("text/uri-list;class=java.lang.String");
+ urlListFlavor = new DataFlavor("text/plain;class=java.lang.String");
+ } catch (ClassNotFoundException e) {
+ //swallow
+ }
}
+ private final TransferHandler delegate;
+ private final TikaGUI tika;
+
public ParsingTransferHandler(TransferHandler delegate, TikaGUI tika) {
this.delegate = delegate;
this.tika = tika;
}
+ private static List<File> uriToFileList(Object data) {
+ List<File> list = new ArrayList<>();
+ StringTokenizer st = new StringTokenizer(data.toString(), "\r\n");
+ while (st.hasMoreTokens()) {
+ String s = st.nextToken();
+ if (s.startsWith("#")) {
+ continue;
+ }
+ try {
+ list.add(new File(new URI(s)));
+ } catch (Exception e) {
+ //swallow
+ }
+ }
+ return list;
+ }
+
public boolean canImport(JComponent component, DataFlavor[] flavors) {
for (DataFlavor flavor : flavors) {
- if (flavor.equals(DataFlavor.javaFileListFlavor) || flavor.equals(uriListFlavor) || flavor.equals(urlListFlavor)) {
+ if (flavor.equals(DataFlavor.javaFileListFlavor) || flavor.equals(uriListFlavor) ||
+ flavor.equals(urlListFlavor)) {
return true;
}
}
@@ -70,18 +87,16 @@ class ParsingTransferHandler extends TransferHandler {
}
@SuppressWarnings("unchecked")
- public boolean importData(
- JComponent component, Transferable transferable) {
+ public boolean importData(JComponent component, Transferable transferable) {
try {
if (transferable.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) {
- importFiles((List<File>) transferable.getTransferData(
- DataFlavor.javaFileListFlavor));
+ importFiles(
+ (List<File>) transferable.getTransferData(DataFlavor.javaFileListFlavor));
} else if (transferable.isDataFlavorSupported(urlListFlavor)) {
Object data = transferable.getTransferData(urlListFlavor);
tika.openURL(new URL(data.toString()));
} else if (transferable.isDataFlavorSupported(uriListFlavor)) {
- importFiles(uriToFileList(
- transferable.getTransferData(uriListFlavor)));
+ importFiles(uriToFileList(transferable.getTransferData(uriListFlavor)));
}
return true;
} catch (Exception e) {
@@ -111,21 +126,4 @@ class ParsingTransferHandler extends TransferHandler {
public Icon getVisualRepresentation(Transferable arg0) {
return delegate.getVisualRepresentation(arg0);
}
-
- private static List<File> uriToFileList(Object data) {
- List<File> list = new ArrayList<>();
- StringTokenizer st = new StringTokenizer(data.toString(), "\r\n");
- while (st.hasMoreTokens())
- {
- String s = st.nextToken();
- if (s.startsWith("#")) {
- continue;
- }
- try {
- list.add(new File(new URI(s)));
- } catch (Exception e) {
- }
- }
- return list;
- }
}
diff --git a/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java b/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
index 386d0d4..f23da18 100644
--- a/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
+++ b/tika-app/src/main/java/org/apache/tika/gui/TikaGUI.java
@@ -16,6 +16,29 @@
*/
package org.apache.tika.gui;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.awt.CardLayout;
+import java.awt.Color;
+import java.awt.Dimension;
+import java.awt.Toolkit;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.awt.event.KeyEvent;
+import java.awt.event.WindowEvent;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
import javax.swing.Box;
import javax.swing.JDialog;
import javax.swing.JEditorPane;
@@ -38,29 +61,13 @@ import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
-import java.awt.CardLayout;
-import java.awt.Color;
-import java.awt.Dimension;
-import java.awt.Toolkit;
-import java.awt.event.ActionEvent;
-import java.awt.event.ActionListener;
-import java.awt.event.KeyEvent;
-import java.awt.event.WindowEvent;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
import org.apache.commons.io.IOUtils;
+import org.xml.sax.Attributes;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
+
import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.DocumentSelector;
@@ -76,26 +83,19 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.digestutils.CommonsDigester;
-import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerDecorator;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
import org.apache.tika.sax.TeeContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.AttributesImpl;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
+import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
/**
* Simple Swing GUI for Apache Tika. You can drag and drop files on top
* of the window to have them parsed.
*/
-public class TikaGUI extends JFrame
- implements ActionListener, HyperlinkListener {
+public class TikaGUI extends JFrame implements ActionListener, HyperlinkListener {
//maximum length to allow for mark for reparse to get JSON
private static final int MAX_MARK = 20 * 1024 * 1024;//20MB
@@ -104,86 +104,50 @@ public class TikaGUI extends JFrame
* Serial version UID.
*/
private static final long serialVersionUID = 5883906936187059495L;
-
- /**
- * Main method. Sets the Swing look and feel to the operating system
- * settings, and starts the Tika GUI with an {@link AutoDetectParser}
- * instance as the default parser.
- *
- * @param args ignored
- * @throws Exception if an error occurs
- */
- public static void main(String[] args) throws Exception {
- TikaConfig config = TikaConfig.getDefaultConfig();
- if (args.length > 0) {
- File configFile = new File(args[0]);
- config = new TikaConfig(configFile);
- }
- UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
- final TikaConfig finalConfig = config;
- SwingUtilities.invokeLater(() -> new TikaGUI(new DigestingParser(
- new AutoDetectParser(finalConfig),
- new CommonsDigester(MAX_MARK,
- CommonsDigester.DigestAlgorithm.MD5,
- CommonsDigester.DigestAlgorithm.SHA256)
- )).setVisible(true));
- }
-
/**
* Parsing context.
*/
private final ParseContext context;
-
/**
* Configured parser instance.
*/
private final Parser parser;
-
/**
* Captures requested embedded images
*/
private final ImageSavingParser imageParser;
-
/**
* The card layout for switching between different views.
*/
private final CardLayout layout = new CardLayout();
-
/**
* Container for the editor cards.
*/
private final JPanel cards;
-
/**
* Formatted XHTML output.
*/
private final JEditorPane html;
-
/**
* Plain text output.
*/
private final JEditorPane text;
-
/**
* Main content output.
*/
private final JEditorPane textMain;
-
/**
* Raw XHTML source.
*/
private final JEditorPane xml;
-
/**
* Raw JSON source.
*/
private final JEditorPane json;
-
/**
* Document metadata.
*/
private final JEditorPane metadata;
-
/**
* File chooser.
*/
@@ -217,6 +181,28 @@ public class TikaGUI extends JFrame
this.context.set(Parser.class, imageParser);
}
+ /**
+ * Main method. Sets the Swing look and feel to the operating system
+ * settings, and starts the Tika GUI with an {@link AutoDetectParser}
+ * instance as the default parser.
+ *
+ * @param args ignored
+ * @throws Exception if an error occurs
+ */
+ public static void main(String[] args) throws Exception {
+ TikaConfig config = TikaConfig.getDefaultConfig();
+ if (args.length > 0) {
+ File configFile = new File(args[0]);
+ config = new TikaConfig(configFile);
+ }
+ UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
+ final TikaConfig finalConfig = config;
+ SwingUtilities.invokeLater(() -> new TikaGUI(
+ new DigestingParser(new AutoDetectParser(finalConfig),
+ new CommonsDigester(MAX_MARK, CommonsDigester.DigestAlgorithm.MD5,
+ CommonsDigester.DigestAlgorithm.SHA256))).setVisible(true));
+ }
+
private void addMenuBar() {
JMenuBar bar = new JMenuBar();
@@ -247,8 +233,7 @@ public class TikaGUI extends JFrame
setJMenuBar(bar);
}
- private void addMenuItem(
- JMenu menu, String title, String command, int key) {
+ private void addMenuItem(JMenu menu, String title, String command, int key) {
JMenuItem item = new JMenuItem(title, key);
item.setActionCommand(command);
item.addActionListener(this);
@@ -263,16 +248,14 @@ public class TikaGUI extends JFrame
openFile(chooser.getSelectedFile());
}
} else if ("openurl".equals(command)) {
- Object rv = JOptionPane.showInputDialog(
- this, "Enter the URL of the resource to be parsed:",
- "Open URL", JOptionPane.PLAIN_MESSAGE,
- null, null, "");
+ Object rv = JOptionPane
+ .showInputDialog(this, "Enter the URL of the resource to be parsed:",
+ "Open URL", JOptionPane.PLAIN_MESSAGE, null, null, "");
if (rv != null && rv.toString().length() > 0) {
try {
openURL(new URL(rv.toString().trim()));
} catch (MalformedURLException exception) {
- JOptionPane.showMessageDialog(
- this, "The given string is not a valid URL",
+ JOptionPane.showMessageDialog(this, "The given string is not a valid URL",
"Invalid URL", JOptionPane.ERROR_MESSAGE);
}
}
@@ -289,12 +272,10 @@ public class TikaGUI extends JFrame
} else if ("json".equals(command)) {
layout.show(cards, command);
} else if ("about".equals(command)) {
- textDialog(
- "About Apache Tika",
- TikaGUI.class.getResource("about.html"));
+ textDialog("About Apache Tika", TikaGUI.class.getResource("about.html"));
} else if ("exit".equals(command)) {
- Toolkit.getDefaultToolkit().getSystemEventQueue().postEvent(
- new WindowEvent(this, WindowEvent.WINDOW_CLOSING));
+ Toolkit.getDefaultToolkit().getSystemEventQueue()
+ .postEvent(new WindowEvent(this, WindowEvent.WINDOW_CLOSING));
}
}
@@ -320,19 +301,16 @@ public class TikaGUI extends JFrame
}
}
- private void handleStream(InputStream input, Metadata md)
- throws Exception {
+ private void handleStream(InputStream input, Metadata md) throws Exception {
StringWriter htmlBuffer = new StringWriter();
StringWriter textBuffer = new StringWriter();
StringWriter textMainBuffer = new StringWriter();
StringWriter xmlBuffer = new StringWriter();
StringBuilder metadataBuffer = new StringBuilder();
- ContentHandler handler = new TeeContentHandler(
- getHtmlHandler(htmlBuffer),
- getTextContentHandler(textBuffer),
- getTextMainContentHandler(textMainBuffer),
- getXmlContentHandler(xmlBuffer));
+ ContentHandler handler =
+ new TeeContentHandler(getHtmlHandler(htmlBuffer), getTextContentHandler(textBuffer),
+ getTextMainContentHandler(textMainBuffer), getXmlContentHandler(xmlBuffer));
context.set(DocumentSelector.class, new ImageDocumentSelector());
@@ -385,17 +363,14 @@ public class TikaGUI extends JFrame
input.reset();
isReset = true;
} catch (IOException e) {
- setText(json, "Error during stream reset.\n" +
- "There's a limit of " + MAX_MARK + " bytes for this type of processing in the GUI.\n" +
- "Try the app with command line argument of -J."
- );
+ setText(json, "Error during stream reset.\n" + "There's a limit of " + MAX_MARK +
+ " bytes for this type of processing in the GUI.\n" +
+ "Try the app with command line argument of -J.");
}
if (isReset) {
RecursiveParserWrapperHandler recursiveParserWrapperHandler =
- new RecursiveParserWrapperHandler(
- new BasicContentHandlerFactory(
- BasicContentHandlerFactory.HANDLER_TYPE.BODY, -1),
- -1);
+ new RecursiveParserWrapperHandler(new BasicContentHandlerFactory(
+ BasicContentHandlerFactory.HANDLER_TYPE.BODY, -1), -1);
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
wrapper.parse(input, recursiveParserWrapperHandler, new Metadata(), new ParseContext());
StringWriter jsonBuffer = new StringWriter();
@@ -413,8 +388,7 @@ public class TikaGUI extends JFrame
writer.append("The full exception stack trace is included below:\n\n");
t.printStackTrace(new PrintWriter(writer));
- JEditorPane editor =
- new JEditorPane("text/plain", writer.toString());
+ JEditorPane editor = new JEditorPane("text/plain", writer.toString());
editor.setEditable(false);
editor.setBackground(Color.WHITE);
editor.setCaretPosition(0);
@@ -428,13 +402,12 @@ public class TikaGUI extends JFrame
private void addWelcomeCard(JPanel panel, String name) {
try {
- JEditorPane editor =
- new JEditorPane(TikaGUI.class.getResource("welcome.html"));
+ JEditorPane editor = new JEditorPane(TikaGUI.class.getResource("welcome.html"));
editor.setContentType("text/html");
editor.setEditable(false);
editor.setBackground(Color.WHITE);
- editor.setTransferHandler(new ParsingTransferHandler(
- editor.getTransferHandler(), this));
+ editor.setTransferHandler(
+ new ParsingTransferHandler(editor.getTransferHandler(), this));
panel.add(new JScrollPane(editor), name);
} catch (IOException e) {
e.printStackTrace();
@@ -445,8 +418,7 @@ public class TikaGUI extends JFrame
JEditorPane editor = new JTextPane();
editor.setBackground(Color.WHITE);
editor.setContentType(type);
- editor.setTransferHandler(new ParsingTransferHandler(
- editor.getTransferHandler(), this));
+ editor.setTransferHandler(new ParsingTransferHandler(editor.getTransferHandler(), this));
panel.add(new JScrollPane(editor), name);
return editor;
}
@@ -520,17 +492,14 @@ public class TikaGUI extends JFrame
* @return HTML content handler
* @throws TransformerConfigurationException if an error occurs
*/
- private ContentHandler getHtmlHandler(Writer writer)
- throws TransformerConfigurationException {
- SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
+ private ContentHandler getHtmlHandler(Writer writer) throws TransformerConfigurationException {
+ SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
handler.setResult(new StreamResult(writer));
return new ContentHandlerDecorator(handler) {
@Override
- public void startElement(
- String uri, String localName, String name, Attributes atts)
+ public void startElement(String uri, String localName, String name, Attributes atts)
throws SAXException {
if (XHTMLContentHandler.XHTML.equals(uri)) {
uri = null;
@@ -554,7 +523,8 @@ public class TikaGUI extends JFrame
String newSrc = img.toURI().toString();
newAttrs.setValue(i, newSrc);
} catch (IOException e) {
- System.err.println("Error creating temp image file " + filename);
+ System.err.println(
+ "Error creating temp image file " + filename);
// The html viewer will show a broken image too to alert them
}
}
@@ -568,8 +538,7 @@ public class TikaGUI extends JFrame
}
@Override
- public void endElement(String uri, String localName, String name)
- throws SAXException {
+ public void endElement(String uri, String localName, String name) throws SAXException {
if (XHTMLContentHandler.XHTML.equals(uri)) {
uri = null;
}
@@ -598,8 +567,7 @@ public class TikaGUI extends JFrame
private ContentHandler getXmlContentHandler(Writer writer)
throws TransformerConfigurationException {
- SAXTransformerFactory factory = (SAXTransformerFactory)
- SAXTransformerFactory.newInstance();
+ SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
handler.setResult(new StreamResult(writer));
@@ -633,6 +601,7 @@ public class TikaGUI extends JFrame
File t = File.createTempFile("tika", ".test");
tmpDir = t.getParentFile();
} catch (IOException e) {
+ //swallow
}
}
@@ -653,9 +622,8 @@ public class TikaGUI extends JFrame
return downstreamParser.getSupportedTypes(context);
}
- public void parse(InputStream stream, ContentHandler handler,
- Metadata metadata, ParseContext context) throws IOException,
- SAXException, TikaException {
+ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
+ ParseContext context) throws IOException, SAXException, TikaException {
String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
if (name != null && wanted.containsKey(name)) {
FileOutputStream out = new FileOutputStream(wanted.get(name));
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchCommandLineTest.java b/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchCommandLineTest.java
index a5f95ea..e48c5fb 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchCommandLineTest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchCommandLineTest.java
@@ -31,11 +31,12 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.tika.utils.ProcessUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.apache.tika.utils.ProcessUtils;
+
public class TikaCLIBatchCommandLineTest {
Path testInput = null;
@@ -59,7 +60,8 @@ public class TikaCLIBatchCommandLineTest {
throw new RuntimeException("Couldn't open testFile");
}
testInputPathForCommandLine = testInput.toAbsolutePath().toString();
- escapedInputPathForCommandLine = BatchCommandLineBuilder.commandLineSafe(testInputPathForCommandLine);
+ escapedInputPathForCommandLine =
+ BatchCommandLineBuilder.commandLineSafe(testInputPathForCommandLine);
}
@After
@@ -105,12 +107,11 @@ public class TikaCLIBatchCommandLineTest {
assertTrue(log < classInd);
assertTrue(inputDir > classInd);
}
-
+
@Test
public void testBasicMappingOfArgs() throws Exception {
- String[] params = {"-JXmx1g", "-JDlog4j.configuration=batch_process_log4j.xml",
- "-bc", "batch-config.xml",
- "-J", "-h", "-inputDir", testInputPathForCommandLine};
+ String[] params = {"-JXmx1g", "-JDlog4j.configuration=batch_process_log4j.xml", "-bc",
+ "batch-config.xml", "-J", "-h", "-inputDir", testInputPathForCommandLine};
String[] commandLine = BatchCommandLineBuilder.build(params);
Map<String, String> attrs = mapify(commandLine);
@@ -163,7 +164,7 @@ public class TikaCLIBatchCommandLineTest {
String configPath = "c:/somewhere/someConfig.xml";
String[] params = {"--inputDir", testInputPathForCommandLine, "--outputDir", outputRoot,
- "--config="+configPath};
+ "--config=" + configPath};
String[] commandLine = BatchCommandLineBuilder.build(params);
Map<String, String> attrs = mapify(commandLine);
assertEquals(escapedInputPathForCommandLine, attrs.get("-inputDir"));
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchIntegrationTest.java b/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchIntegrationTest.java
index f4ada06..6870d45 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchIntegrationTest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLIBatchIntegrationTest.java
@@ -33,19 +33,18 @@ import java.nio.file.Paths;
import java.util.List;
import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.serialization.JsonMetadataList;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
public class TikaCLIBatchIntegrationTest {
- private Path testInputDir = Paths.get("src/test/resources/test-data");
private final String propsFileName = "log4j2_batch_process_test.properties";
+ private Path testInputDir = Paths.get("src/test/resources/test-data");
private String testInputDirForCommandLine;
private Path tempOutputDir;
private String tempOutputDirForCommandLine;
@@ -81,8 +80,7 @@ public class TikaCLIBatchIntegrationTest {
@Test
public void testSimplestBatchIntegration() throws Exception {
- String[] params = {testInputDirForCommandLine,
- tempOutputDirForCommandLine};
+ String[] params = {testInputDirForCommandLine, tempOutputDirForCommandLine};
TikaCLI.main(params);
assertFileExists(tempOutputDir.resolve("bad_xml.xml.xml"));
@@ -91,10 +89,8 @@ public class TikaCLIBatchIntegrationTest {
@Test
public void testBasicBatchIntegration() throws Exception {
- String[] params = {"-i", testInputDirForCommandLine,
- "-o", tempOutputDirForCommandLine,
- "-numConsumers", "2"
- };
+ String[] params = {"-i", testInputDirForCommandLine, "-o", tempOutputDirForCommandLine,
+ "-numConsumers", "2"};
TikaCLI.main(params);
assertFileExists(tempOutputDir.resolve("bad_xml.xml.xml"));
@@ -103,10 +99,8 @@ public class TikaCLIBatchIntegrationTest {
@Test
public void testJsonRecursiveBatchIntegration() throws Exception {
- String[] params = {"-i", testInputDirForCommandLine,
- "-o", tempOutputDirForCommandLine,
- "-numConsumers", "10",
- "-J", //recursive Json
+ String[] params = {"-i", testInputDirForCommandLine, "-o", tempOutputDirForCommandLine,
+ "-numConsumers", "10", "-J", //recursive Json
"-t" //plain text in content
};
TikaCLI.main(params);
@@ -115,28 +109,27 @@ public class TikaCLIBatchIntegrationTest {
try (Reader reader = Files.newBufferedReader(jsonFile, UTF_8)) {
List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
assertEquals(12, metadataList.size());
- assertTrue(metadataList.get(6).get(TikaCoreProperties.TIKA_CONTENT).contains("human events"));
+ assertTrue(metadataList.get(6).get(TikaCoreProperties.TIKA_CONTENT)
+ .contains("human events"));
}
}
@Test
public void testStreamingJsonRecursiveBatchIntegration() throws Exception {
- String[] params = {"-i", testInputDirForCommandLine,
- "-o", tempOutputDirForCommandLine,
- "-numConsumers", "10",
- "-J", //recursive Json
+ String[] params = {"-i", testInputDirForCommandLine, "-o", tempOutputDirForCommandLine,
+ "-numConsumers", "10", "-J", //recursive Json
"-t", //plain text in content
- "-streamOut"
- };
+ "-streamOut"};
TikaCLI.main(params);
Path jsonFile = tempOutputDir.resolve("test_recursive_embedded.docx.json");
try (Reader reader = Files.newBufferedReader(jsonFile, UTF_8)) {
List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
assertEquals(12, metadataList.size());
- assertTrue(metadataList.get(6).get(TikaCoreProperties.TIKA_CONTENT).contains("human events"));
+ assertTrue(metadataList.get(6).get(TikaCoreProperties.TIKA_CONTENT)
+ .contains("human events"));
//test that the last written object has been bumped to the first by JsonMetadataList.fromJson()
- assertNull( metadataList.get(0).get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
+ assertNull(metadataList.get(0).get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH));
}
}
@@ -173,13 +166,10 @@ public class TikaCLIBatchIntegrationTest {
IOUtils.closeQuietly(reader);
}
*/
- String[] params = {"-i", testInputDirForCommandLine,
- "-o", tempOutputDirForCommandLine,
- "-numConsumers", "10",
- "-J", //recursive Json
+ String[] params = {"-i", testInputDirForCommandLine, "-o", tempOutputDirForCommandLine,
+ "-numConsumers", "10", "-J", //recursive Json
"-t", //plain text in content
- "-digest", "sha512"
- };
+ "-digest", "sha512"};
TikaCLI.main(params);
Path jsonFile = tempOutputDir.resolve("test_recursive_embedded.docx.json");
try (Reader reader = Files.newBufferedReader(jsonFile, UTF_8)) {
@@ -187,13 +177,13 @@ public class TikaCLIBatchIntegrationTest {
List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
assertEquals(12, metadataList.size());
assertNotNull(metadataList.get(0).get("X-TIKA:digest:SHA512"));
- assertTrue(metadataList.get(0).get("X-TIKA:digest:SHA512").startsWith("ee46d973ee1852c01858"));
+ assertTrue(metadataList.get(0).get("X-TIKA:digest:SHA512")
+ .startsWith("ee46d973ee1852c01858"));
}
}
private void assertFileExists(Path path) {
- assertTrue("File doesn't exist: "+path.toAbsolutePath(),
- Files.isRegularFile(path));
+ assertTrue("File doesn't exist: " + path.toAbsolutePath(), Files.isRegularFile(path));
}
}
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
index 1ede66b..c8ec19b 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
@@ -30,64 +30,47 @@ import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.commons.io.FileUtils;
-import org.apache.tika.exception.TikaException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.tika.exception.TikaException;
+
/**
* Tests the Tika's cli
*/
public class TikaCLITest {
+ private static final File testDataFile = new File("src/test/resources/test-data");
+ private static Path ASYNC_CONFIG;
+ private static Path ASYNC_OUTPUT_DIR;
+ private final URI testDataURI = testDataFile.toURI();
/* Test members */
private ByteArrayOutputStream outContent = null;
private ByteArrayOutputStream errContent = null;
private PrintStream stdout = null;
private PrintStream stderr = null;
- private static final File testDataFile = new File("src/test/resources/test-data");
- private final URI testDataURI = testDataFile.toURI();
private String resourcePrefix;
- private static Path ASYNC_CONFIG;
- private static Path ASYNC_OUTPUT_DIR;
@BeforeClass
public static void setUpClass() throws Exception {
ASYNC_OUTPUT_DIR = Files.createTempDirectory("tika-cli-async-");
ASYNC_CONFIG = Files.createTempFile("async-config-", ".xml");
- String xml = "<properties>" +
- "<async>" +
- "<params>" +
- "<numClients>3</numClients>" +
- "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" +
- "</params>" +
- "</async>" +
- "<fetchers>" +
+ String xml = "<properties>" + "<async>" + "<params>" + "<numClients>3</numClients>" +
+ "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" + "</params>" +
+ "</async>" + "<fetchers>" +
"<fetcher class=\"org.apache.tika.pipes.fetcher.fs.FileSystemFetcher\">" +
- "<params>" +
- "<name>fsf</name>" +
- "<basePath>" + testDataFile.getAbsolutePath() + "</basePath>" +
- "</params>" +
- "</fetcher>" +
- "</fetchers>" +
- "<emitters>" +
+ "<params>" + "<name>fsf</name>" + "<basePath>" + testDataFile.getAbsolutePath() +
+ "</basePath>" + "</params>" + "</fetcher>" + "</fetchers>" + "<emitters>" +
"<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" +
- "<params>" +
- "<name>fse</name>" +
- "<basePath>" + ASYNC_OUTPUT_DIR.toAbsolutePath() + "</basePath>" +
- "</params></emitter>" +
- "</emitters>" +
- "<pipesIterator " +
+ "<params>" + "<name>fse</name>" + "<basePath>" + ASYNC_OUTPUT_DIR.toAbsolutePath() +
+ "</basePath>" + "</params></emitter>" + "</emitters>" + "<pipesIterator " +
"class=\"org.apache.tika.pipes.pipesiterator.FileSystemPipesIterator\">" +
- "<params>" +
- "<basePath>" + testDataFile.getAbsolutePath() + "</basePath>" +
- "<fetcherName>fsf</fetcherName>" +
- "<emitterName>fse</emitterName>" +
- "</params>" +
- "</pipesIterator>" +
- "</properties>";
+ "<params>" + "<basePath>" + testDataFile.getAbsolutePath() + "</basePath>" +
+ "<fetcherName>fsf</fetcherName>" + "<emitterName>fse</emitterName>" + "</params>" +
+ "</pipesIterator>" + "</properties>";
Files.write(ASYNC_CONFIG, xml.getBytes(UTF_8));
}
@@ -97,6 +80,15 @@ public class TikaCLITest {
FileUtils.deleteDirectory(ASYNC_OUTPUT_DIR.toFile());
}
+ protected static void assertExtracted(File f, String allFiles) {
+
+ assertTrue("File " + f.getName() + " not found in " + allFiles, f.exists());
+
+ assertFalse("File " + f.getName() + " is a directory!", f.isDirectory());
+
+ assertTrue("File " + f.getName() + " wasn't extracted with contents", f.length() > 0);
+ }
+
/**
* reset resourcePrefix
* save original System.out and System.err
@@ -153,7 +145,7 @@ public class TikaCLITest {
/**
* Tests --list-parser option of the cli
- *
+ *
* @throws Exception
*/
@Test
@@ -164,7 +156,7 @@ public class TikaCLITest {
/**
* Tests -x option of the cli
- *
+ *
* @throws Exception
*/
@Test
@@ -173,28 +165,31 @@ public class TikaCLITest {
assertTrue(content.contains("?xml version=\"1.0\" encoding=\"UTF-8\"?"));
content = getParamOutContent("-x", "--digest=SHA256", resourcePrefix + "alice.cli.test");
- assertTrue(content.contains("<meta name=\"X-TIKA:digest:SHA256\" content=\"e90779adbac09c4ee"));
+ assertTrue(content.contains(
+ "<meta name=\"X-TIKA:digest:SHA256\" content=\"e90779adbac09c4ee"));
}
/**
* Tests a -h option of the cli
- *
+ *
* @throws Exception
*/
@Test
public void testHTMLOutput() throws Exception {
String content = getParamOutContent("-h", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("html xmlns=\"http://www.w3.org/1999/xhtml"));
- assertTrue("Expanded <title></title> element should be present", content.contains("<title></title>"));
+ assertTrue("Expanded <title></title> element should be present",
+ content.contains("<title></title>"));
content = getParamOutContent("-h", "--digest=SHA384", resourcePrefix + "alice.cli.test");
- assertTrue(content.contains("<meta name=\"X-TIKA:digest:SHA384\" content=\"c69ea023f5da95a026"));
+ assertTrue(content.contains(
+ "<meta name=\"X-TIKA:digest:SHA384\" content=\"c69ea023f5da95a026"));
}
/**
* Tests -t option of the cli
- *
+ *
* @throws Exception
*/
@Test
@@ -209,7 +204,7 @@ public class TikaCLITest {
* @throws Exception
*/
@Test
- public void testContentAllOutput() throws Exception{
+ public void testContentAllOutput() throws Exception {
String[] params = {"-A", resourcePrefix + "testJsonMultipleInts.html"};
TikaCLI.main(params);
String out = outContent.toString(UTF_8.name());
@@ -227,8 +222,10 @@ public class TikaCLITest {
String content = getParamOutContent("-f", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("finished off the cake"));
}
+
/**
* Tests -m option of the cli
+ *
* @throws Exception
*/
@Test
@@ -243,21 +240,22 @@ public class TikaCLITest {
/**
* Basic tests for -json option
- *
+ *
* @throws Exception
*/
@Test
public void testJsonMetadataOutput() throws Exception {
- String json = getParamOutContent("--json", "--digest=MD2", resourcePrefix + "testJsonMultipleInts.html");
+ String json = getParamOutContent("--json", "--digest=MD2",
+ resourcePrefix + "testJsonMultipleInts.html");
//TIKA-1310
assertTrue(json.contains("\"fb:admins\":\"1,2,3,4\","));
-
+
//test legacy alphabetic sort of keys
int enc = json.indexOf("\"Content-Encoding\"");
int fb = json.indexOf("fb:admins");
int title = json.indexOf("\"dc:title\"");
assertTrue(enc > -1 && fb > -1 && enc < fb);
- assertTrue (fb > -1 && title > -1 && fb > title);
+ assertTrue(fb > -1 && title > -1 && fb > title);
assertTrue(json.contains("\"X-TIKA:digest:MD2\":"));
}
@@ -268,21 +266,23 @@ public class TikaCLITest {
*/
@Test
public void testJsonMetadataPrettyPrintOutput() throws Exception {
- String json = getParamOutContent("--json", "-r", resourcePrefix + "testJsonMultipleInts.html");
+ String json =
+ getParamOutContent("--json", "-r", resourcePrefix + "testJsonMultipleInts.html");
- assertTrue(json.contains("\"X-TIKA:Parsed-By\" : [ \"org.apache.tika.parser.DefaultParser\", " +
- "\"org.apache.tika.parser.html.HtmlParser\" ],"));
+ assertTrue(json.contains(
+ "\"X-TIKA:Parsed-By\" : [ \"org.apache.tika.parser.DefaultParser\", " +
+ "\"org.apache.tika.parser.html.HtmlParser\" ],"));
//test legacy alphabetic sort of keys
int enc = json.indexOf("\"Content-Encoding\"");
int fb = json.indexOf("fb:admins");
int title = json.indexOf("\"dc:title\"");
assertTrue(enc > -1 && fb > -1 && enc < fb);
- assertTrue (fb > -1 && title > -1 && fb > title);
+ assertTrue(fb > -1 && title > -1 && fb > title);
}
/**
* Tests -l option of the cli
- *
+ *
* @throws Exception
*/
@Test
@@ -293,7 +293,7 @@ public class TikaCLITest {
/**
* Tests -d option of the cli
- *
+ *
* @throws Exception
*/
@Test
@@ -304,7 +304,7 @@ public class TikaCLITest {
/**
* Tests --list-met-models option of the cli
- *
+ *
* @throws Exception
*/
@Test
@@ -315,70 +315,63 @@ public class TikaCLITest {
/**
* Tests --list-supported-types option of the cli
- *
+ *
* @throws Exception
*/
@Test
public void testListSupportedTypes() throws Exception {
- String content = getParamOutContent("--list-supported-types", resourcePrefix + "alice.cli.test");
+ String content =
+ getParamOutContent("--list-supported-types", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("supertype: application/octet-stream"));
}
@Test
public void testExtractSimple() throws Exception {
- String[] expectedChildren = new String[]{
- "MBD002B040A.cdx",
- "file4.png",
- "MBD002B0FA6_file5.bin",
- "MBD00262FE3.txt",
- "file0.emf"
- };
+ String[] expectedChildren =
+ new String[]{"MBD002B040A.cdx", "file4.png", "MBD002B0FA6_file5.bin",
+ "MBD00262FE3.txt", "file0.emf"};
testExtract("/coffee.xls", expectedChildren, 8);
}
@Test
public void testExtractAbsolute() throws Exception {
- String[] expectedChildren = new String[] {
- "dangerous/dont/touch.pl",
- };
+ String[] expectedChildren = new String[]{"dangerous/dont/touch.pl",};
testExtract("testZip_absolutePath.zip", expectedChildren, 2);
}
@Test
public void testExtractRelative() throws Exception {
- String[] expectedChildren = new String[] {
- "touch.pl",
- };
+ String[] expectedChildren = new String[]{"touch.pl",};
testExtract("testZip_relative.zip", expectedChildren);
}
@Test
public void testExtractOverlapping() throws Exception {
//there should be two files, one with a prepended uuid-f1.txt
- String[] expectedChildren = new String[] {
- "f1.txt",
- };
+ String[] expectedChildren = new String[]{"f1.txt",};
testExtract("testZip_overlappingNames.zip", expectedChildren, 2);
}
@Test
public void testExtract0x00() throws Exception {
- String[] expectedChildren = new String[] {
- "dang erous.pl",
- };
+ String[] expectedChildren = new String[]{"dang erous.pl",};
testExtract("testZip_zeroByte.zip", expectedChildren);
}
- private void testExtract(String targetFile, String[] expectedChildrenFileNames) throws Exception {
+ private void testExtract(String targetFile, String[] expectedChildrenFileNames)
+ throws Exception {
testExtract(targetFile, expectedChildrenFileNames, expectedChildrenFileNames.length);
}
- private void testExtract(String targetFile, String[] expectedChildrenFileNames, int expectedLength) throws Exception {
+
+ private void testExtract(String targetFile, String[] expectedChildrenFileNames,
+ int expectedLength) throws Exception {
File tempFile = File.createTempFile("tika-test-", "");
assertTrue(tempFile.delete());
assertTrue(tempFile.mkdir());
try {
- String[] params = {"--extract-dir=" + tempFile.getAbsolutePath(), "-z", resourcePrefix + "/"+targetFile};
+ String[] params = {"--extract-dir=" + tempFile.getAbsolutePath(), "-z",
+ resourcePrefix + "/" + targetFile};
TikaCLI.main(params);
@@ -403,7 +396,8 @@ public class TikaCLITest {
assertTrue(tempFile.mkdir());
try {
- String[] params = {"--extract-dir="+tempFile.getAbsolutePath(),"-z", resourcePrefix + "/test-documents.tgz"};
+ String[] params = {"--extract-dir=" + tempFile.getAbsolutePath(), "-z",
+ resourcePrefix + "/test-documents.tgz"};
TikaCLI.main(params);
@@ -419,24 +413,6 @@ public class TikaCLITest {
}
}
-
- protected static void assertExtracted(File f, String allFiles) {
-
- assertTrue(
- "File " + f.getName() + " not found in " + allFiles,
- f.exists()
- );
-
- assertFalse(
- "File " + f.getName() + " is a directory!", f.isDirectory()
- );
-
- assertTrue(
- "File " + f.getName() + " wasn't extracted with contents",
- f.length() > 0
- );
- }
-
// TIKA-920
@Test
public void testMultiValuedMetadata() throws Exception {
@@ -452,7 +428,8 @@ public class TikaCLITest {
public void testZipWithSubdirs() throws Exception {
new File("subdir/foo.txt").delete();
new File("subdir").delete();
- String content = getParamOutContent("-z", "--extract-dir=target", resourcePrefix + "testWithSubdirs.zip");
+ String content = getParamOutContent("-z", "--extract-dir=target",
+ resourcePrefix + "testWithSubdirs.zip");
assertTrue(content.contains("Extracting 'subdir/foo.txt'"));
// clean up. TODO: These should be in target.
new File("target/subdir/foo.txt").delete();
@@ -467,7 +444,8 @@ public class TikaCLITest {
// google guava library has better solution
try {
- String[] params = {"--extract-dir="+tempFile.getAbsolutePath(),"-z", resourcePrefix + "/testPDF_childAttachments.pdf"};
+ String[] params = {"--extract-dir=" + tempFile.getAbsolutePath(), "-z",
+ resourcePrefix + "/testPDF_childAttachments.pdf"};
TikaCLI.main(params);
@@ -506,16 +484,18 @@ public class TikaCLITest {
@Test
public void testConfig() throws Exception {
- String content = getParamOutContent("--config="+testDataFile.toString()+"/tika-config1.xml",
- resourcePrefix+"bad_xml.xml");
+ String content =
+ getParamOutContent("--config=" + testDataFile.toString() + "/tika-config1.xml",
+ resourcePrefix + "bad_xml.xml");
assertTrue(content.contains("apple"));
assertTrue(content.contains("org.apache.tika.parser.html.HtmlParser"));
}
@Test
public void testConfigIgnoreInit() throws Exception {
- String content = getParamOutContent("--config="+testDataFile.toString()+"/TIKA-2389-ignore-init-problems.xml",
- resourcePrefix+"test_recursive_embedded.docx");
+ String content = getParamOutContent(
+ "--config=" + testDataFile.toString() + "/TIKA-2389-ignore-init-problems.xml",
+ resourcePrefix + "test_recursive_embedded.docx");
assertTrue(content.contains("embed_1a"));
//TODO: add a real unit test that configures logging to a file to test that nothing is
//written at the various logging levels
@@ -524,9 +504,9 @@ public class TikaCLITest {
@Test
public void testJsonRecursiveMetadataParserMetadataOnly() throws Exception {
- String content = getParamOutContent("-m", "-J", "-r", resourcePrefix+"test_recursive_embedded.docx");
- assertTrue(content.contains(
- "\"extended-properties:AppVersion\" : \"15.0000\","));
+ String content = getParamOutContent("-m", "-J", "-r",
+ resourcePrefix + "test_recursive_embedded.docx");
+ assertTrue(content.contains("\"extended-properties:AppVersion\" : \"15.0000\","));
assertTrue(content.contains(
"\"extended-properties:Application\" : \"Microsoft Office Word\","));
assertTrue(content.contains("\"X-TIKA:embedded_resource_path\" : \"/embed1.zip\""));
@@ -535,13 +515,16 @@ public class TikaCLITest {
@Test
public void testJsonRecursiveMetadataParserDefault() throws Exception {
- String content = getParamOutContent("-J", "-r", resourcePrefix+"test_recursive_embedded.docx");
- assertTrue(content.contains("\"X-TIKA:content\" : \"<html xmlns=\\\"http://www.w3.org/1999/xhtml"));
+ String content =
+ getParamOutContent("-J", "-r", resourcePrefix + "test_recursive_embedded.docx");
+ assertTrue(content.contains(
+ "\"X-TIKA:content\" : \"<html xmlns=\\\"http://www.w3.org/1999/xhtml"));
}
@Test
public void testJsonRecursiveMetadataParserText() throws Exception {
- String content = getParamOutContent("-J", "-r", "-t", resourcePrefix+"test_recursive_embedded.docx");
+ String content = getParamOutContent("-J", "-r", "-t",
+ resourcePrefix + "test_recursive_embedded.docx");
assertTrue(content.contains("\\n\\nembed_4\\n"));
assertTrue(content.contains("\\n\\nembed_0"));
}
@@ -549,48 +532,51 @@ public class TikaCLITest {
@Test
public void testDigestInJson() throws Exception {
String content = getParamOutContent("-J", "-r", "-t", "--digest=MD5",
- resourcePrefix+"test_recursive_embedded.docx");
- assertTrue(content.contains("\"X-TIKA:digest:MD5\" : \"59f626e09a8c16ab6dbc2800c685f772\","));
- assertTrue(content.contains("\"X-TIKA:digest:MD5\" : \"f9627095ef86c482e61d99f0cc1cf87d\""));
+ resourcePrefix + "test_recursive_embedded.docx");
+ assertTrue(
+ content.contains("\"X-TIKA:digest:MD5\" : \"59f626e09a8c16ab6dbc2800c685f772\","));
+ assertTrue(
+ content.contains("\"X-TIKA:digest:MD5\" : \"f9627095ef86c482e61d99f0cc1cf87d\""));
}
@Test
public void testConfigSerializationStaticAndCurrent() throws Exception {
String content = getParamOutContent("--dump-static-config");
//make sure at least one detector is there
- assertTrue(content.contains("<detector class=\"org.apache.tika.detect.microsoft.POIFSContainerDetector\"/>"));
+ assertTrue(content.contains(
+ "<detector class=\"org.apache.tika.detect.microsoft.POIFSContainerDetector\"/>"));
//make sure Executable is there because follow on tests of custom config
//test that it has been turned off.
- assertTrue(content.contains("<parser class=\"org.apache.tika.parser.executable.ExecutableParser\"/>"));
+ assertTrue(content.contains(
+ "<parser class=\"org.apache.tika.parser.executable.ExecutableParser\"/>"));
content = getParamOutContent("--dump-current-config");
//make sure at least one detector is there
- assertTrue(content.contains("<detector class=\"org.apache.tika.detect.DefaultDetector\"/>"));
+ assertTrue(
+ content.contains("<detector class=\"org.apache.tika.detect.DefaultDetector\"/>"));
//and at least one parser
assertTrue(content.contains("<parser class=\"org.apache.tika.parser.DefaultParser\"/>"));
}
@Test
public void testConfigSerializationCustomMinimal() throws Exception {
- String content = getParamOutContent("--config=" + testDataFile.toString() + "/tika-config2.xml",
- "--dump-minimal-config")
- .replaceAll("[\r\n\t ]+", " ");
-
- String expected =
- "<parser class=\"org.apache.tika.parser.DefaultParser\">" +
- " <mime-exclude>application/pdf</mime-exclude>" +
- " <mime-exclude>image/jpeg</mime-exclude> " +
- "</parser> " +
- "<parser class=\"org.apache.tika.parser.EmptyParser\">" +
- " <mime>application/pdf</mime> " +
- "</parser>";
+ String content =
+ getParamOutContent("--config=" + testDataFile.toString() + "/tika-config2.xml",
+ "--dump-minimal-config").replaceAll("[\r\n\t ]+", " ");
+
+ String expected = "<parser class=\"org.apache.tika.parser.DefaultParser\">" +
+ " <mime-exclude>application/pdf</mime-exclude>" +
+ " <mime-exclude>image/jpeg</mime-exclude> " + "</parser> " +
+ "<parser class=\"org.apache.tika.parser.EmptyParser\">" +
+ " <mime>application/pdf</mime> " + "</parser>";
assertTrue(content.contains(expected));
}
@Test
public void testConfigSerializationCustomStatic() throws Exception {
- String content = getParamOutContent("--config=" + testDataFile.toString() + "/tika-config2.xml",
- "--dump-static-config");
+ String content =
+ getParamOutContent("--config=" + testDataFile.toString() + "/tika-config2.xml",
+ "--dump-static-config");
assertFalse(content.contains("org.apache.tika.parser.executable.Executable"));
}
@@ -626,8 +612,7 @@ public class TikaCLITest {
@Test
public void testAsync() throws Exception {
- String content = getParamOutContent("-a",
- "--config="+ASYNC_CONFIG.toAbsolutePath());
+ String content = getParamOutContent("-a", "--config=" + ASYNC_CONFIG.toAbsolutePath());
int json = 0;
for (File f : ASYNC_OUTPUT_DIR.toFile().listFiles()) {
diff --git a/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java b/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java
index 8498ee7..593b6db 100644
--- a/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java
+++ b/tika-app/src/test/java/org/apache/tika/extractor/TestEmbeddedDocumentUtil.java
@@ -17,16 +17,17 @@
package org.apache.tika.extractor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.junit.Test;
+
import org.apache.tika.batch.DigestingAutoDetectParserFactory;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.RecursiveParserWrapper;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
public class TestEmbeddedDocumentUtil {
//TODO -- figure out how to mock this into tika-core
@@ -36,7 +37,9 @@ public class TestEmbeddedDocumentUtil {
Parser p = new AutoDetectParser();
ParseContext parseContext = new ParseContext();
parseContext.set(Parser.class, p);
- Parser txtParser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(org.apache.tika.parser.csv.TextAndCSVParser.class, parseContext);
+ Parser txtParser = EmbeddedDocumentUtil
+ .tryToFindExistingLeafParser(org.apache.tika.parser.csv.TextAndCSVParser.class,
+ parseContext);
assertNotNull(txtParser);
assertEquals(org.apache.tika.parser.csv.TextAndCSVParser.class, txtParser.getClass());
@@ -45,10 +48,12 @@ public class TestEmbeddedDocumentUtil {
@Test
public void testDoublyDecorated() {
Parser d = new DigestingAutoDetectParserFactory().getParser(TikaConfig.getDefaultConfig());
- RecursiveParserWrapper wrapper = new RecursiveParserWrapper(d,true);
+ RecursiveParserWrapper wrapper = new RecursiveParserWrapper(d, true);
ParseContext parseContext = new ParseContext();
parseContext.set(Parser.class, wrapper);
- Parser txtParser = EmbeddedDocumentUtil.tryToFindExistingLeafParser(org.apache.tika.parser.csv.TextAndCSVParser.class, parseContext);
+ Parser txtParser = EmbeddedDocumentUtil
+ .tryToFindExistingLeafParser(org.apache.tika.parser.csv.TextAndCSVParser.class,
+ parseContext);
assertNotNull(txtParser);
assertEquals(org.apache.tika.parser.csv.TextAndCSVParser.class, txtParser.getClass());
}
diff --git a/tika-app/src/test/resources/log4j2_batch_process_test.properties b/tika-app/src/test/resources/log4j2_batch_process_test.properties
index a99ee2c..ce40c70 100644
--- a/tika-app/src/test/resources/log4j2_batch_process_test.properties
+++ b/tika-app/src/test/resources/log4j2_batch_process_test.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 3656ced..d8c409f 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -605,7 +605,7 @@
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<testSourceDirectories>${project.basedir}/src/test/java</testSourceDirectories>
<violationSeverity>warn</violationSeverity>
- <failOnViolation>false</failOnViolation>
+ <failOnViolation>true</failOnViolation>
</configuration>
<goals>
<goal>check</goal>