You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2019/08/16 22:34:18 UTC
[tika] branch master updated (8bc50ec -> e939504)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git.
from 8bc50ec TIKA-2906 -- add examples for textstats
new 8257e85 TIKA-2924 -- tika server should return 400 for bad headers, not 500
new eebd150 TIKA-2925 upgrade jackson databind to 2.9.9.2
new dad1c72 TIKA-2926 -- add property for container exception in RPW
new e287fe2 TIKA-2906 -- cleanup and bug fix on entropy
new e939504 improve logging in tika-server
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../RecursiveParserWrapperFSConsumerTest.java | 3 +-
.../apache/tika/parser/RecursiveParserWrapper.java | 2 +-
.../sax/AbstractRecursiveParserWrapperHandler.java | 5 +++-
.../org/apache/tika/eval/AbstractProfiler.java | 13 ++-------
.../apache/tika/eval/textstats/TokenEntropy.java | 2 +-
.../apache/tika/eval/util/EvalExceptionUtils.java} | 34 ++++++++++++----------
tika-parent/pom.xml | 2 +-
.../apache/tika/server/ServerStatusWatcher.java | 8 +++--
.../apache/tika/server/resource/TikaResource.java | 6 ++--
.../org/apache/tika/server/TikaResourceTest.java | 4 +--
10 files changed, 42 insertions(+), 37 deletions(-)
copy tika-eval/src/{test/java/org/apache/tika/eval/util/LanguageIdTest.java => main/java/org/apache/tika/eval/util/EvalExceptionUtils.java} (52%)
[tika] 02/05: TIKA-2925 upgrade jackson databind to 2.9.9.2
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit eebd150e0a7edd56223c3b8c320ec313196608a8
Author: TALLISON <ta...@apache.org>
AuthorDate: Fri Aug 16 12:24:13 2019 -0400
TIKA-2925 upgrade jackson databind to 2.9.9.2
---
tika-parent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index e64be44..fdb478a 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -345,7 +345,7 @@
<cxf.version>3.3.2</cxf.version>
<slf4j.version>1.7.26</slf4j.version>
<jackson.version>2.9.9</jackson.version>
- <jackson.databind.version>2.9.9.1</jackson.databind.version>
+ <jackson.databind.version>2.9.9.2</jackson.databind.version>
<!-- when this is next upgraded, see if we can get rid of
javax.activation dependency in tika-server -->
<jaxb.version>2.3.2</jaxb.version>
[tika] 01/05: TIKA-2924 -- tika server should return 400 for bad
headers, not 500
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 8257e85ae0849e4ce120d5e1be8a91afac5b557c
Author: TALLISON <ta...@apache.org>
AuthorDate: Fri Aug 16 12:22:16 2019 -0400
TIKA-2924 -- tika server should return 400 for bad headers, not 500
---
.../src/main/java/org/apache/tika/server/resource/TikaResource.java | 6 ++++--
.../src/test/java/org/apache/tika/server/TikaResourceTest.java | 4 ++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
index 2eb3baf..2a85305 100644
--- a/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
+++ b/tika-server/src/main/java/org/apache/tika/server/resource/TikaResource.java
@@ -265,8 +265,10 @@ public class TikaResource {
}
} catch (Throwable ex) {
- throw new WebApplicationException(String.format(Locale.ROOT,
- "%s is an invalid %s header", key, X_TIKA_OCR_HEADER_PREFIX));
+ throw new WebApplicationException(
+ String.format(Locale.ROOT,
+ "%s is an invalid %s header",
+ key, X_TIKA_OCR_HEADER_PREFIX), Response.Status.BAD_REQUEST);
}
}
diff --git a/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java b/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
index ef97629..0f46f66 100644
--- a/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
+++ b/tika-server/src/test/java/org/apache/tika/server/TikaResourceTest.java
@@ -286,7 +286,7 @@ public class TikaResourceTest extends CXFTestBase {
.accept("text/plain")
.header(TikaResource.X_TIKA_PDF_HEADER_PREFIX + "OcrStrategy", "non-sense-value")
.put(ClassLoader.getSystemResourceAsStream("testOCR.pdf"));
- assertEquals(500, response.getStatus());
+ assertEquals(400, response.getStatus());
}
//TIKA-2669
@@ -385,7 +385,7 @@ public class TikaResourceTest extends CXFTestBase {
"trustedPageSeparator",
"\u0020")
.put(ClassLoader.getSystemResourceAsStream("testOCR.pdf"));
- assertEquals(500, response.getStatus());
+ assertEquals(400, response.getStatus());
}
[tika] 04/05: TIKA-2906 -- cleanup and bug fix on entropy
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit e287fe297a113bc172f1414b6c9cf5bdd91af307
Author: TALLISON <ta...@apache.org>
AuthorDate: Fri Aug 16 12:53:48 2019 -0400
TIKA-2906 -- cleanup and bug fix on entropy
---
.../org/apache/tika/eval/AbstractProfiler.java | 13 ++------
.../apache/tika/eval/textstats/TokenEntropy.java | 2 +-
.../EvalExceptionUtils.java} | 35 ++++++++++++----------
3 files changed, 23 insertions(+), 27 deletions(-)
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
index becccf3..e23361b 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
@@ -66,6 +66,7 @@ import org.apache.tika.eval.tokens.TokenCounts;
import org.apache.tika.eval.tokens.TokenIntPair;
import org.apache.tika.eval.util.ContentTagParser;
import org.apache.tika.eval.util.ContentTags;
+import org.apache.tika.eval.util.EvalExceptionUtils;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.PagedText;
@@ -183,10 +184,6 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
int maxTokens = 200000;
- //these remove runtime info from the stacktraces so
- //that actual causes can be counted.
- private final static Pattern CAUSED_BY_SNIPPER =
- Pattern.compile("(Caused by: [^:]+):[^\\r\\n]+");
private final static Pattern ACCESS_PERMISSION_EXCEPTION =
Pattern.compile("org\\.apache\\.tika\\.exception\\.AccessPermissionException");
@@ -495,7 +492,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
void getExceptionStrings(Metadata metadata, Map<Cols, String> data) {
- String fullTrace = metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_PREFIX + "runtime");
+ String fullTrace = metadata.get(RecursiveParserWrapperHandler.CONTAINER_EXCEPTION);
if (fullTrace == null) {
fullTrace = metadata.get(AbstractRecursiveParserWrapperHandler.EMBEDDED_EXCEPTION);
@@ -528,11 +525,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
//IOException from org.apache.tika.parser.microsoft.OfficeParser@2b1ea6ee
//For reporting purposes, let's snip off the object id so that we can more
//easily count exceptions.
- String sortTrace = ExceptionUtils.trimMessage(fullTrace);
-
- matcher = CAUSED_BY_SNIPPER.matcher(sortTrace);
- sortTrace = matcher.replaceAll("$1");
- sortTrace = sortTrace.replaceAll("org.apache.tika.", "o.a.t.");
+ String sortTrace = EvalExceptionUtils.normalize(fullTrace);
data.put(Cols.SORT_STACK_TRACE, sortTrace);
}
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
index e9af315..dd538b0 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
@@ -34,6 +34,6 @@ public class TokenEntropy implements TokenCountStatsCalculator<Double> {
p = (double) termFreq / totalTokens;
ent += p * FastMath.log(base, p);
}
- return ent;
+ return -1.0*ent;
}
}
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java b/tika-eval/src/main/java/org/apache/tika/eval/util/EvalExceptionUtils.java
similarity index 50%
copy from tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
copy to tika-eval/src/main/java/org/apache/tika/eval/util/EvalExceptionUtils.java
index e9af315..ecad26f 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/textstats/TokenEntropy.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/util/EvalExceptionUtils.java
@@ -14,26 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.eval.textstats;
+package org.apache.tika.eval.util;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.math3.util.FastMath;
-import org.apache.tika.eval.tokens.TokenCounts;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
-public class TokenEntropy implements TokenCountStatsCalculator<Double> {
+import org.apache.commons.lang3.StringUtils;
+import org.apache.tika.utils.ExceptionUtils;
- @Override
- public Double calculate(TokenCounts tokenCounts) {
- double ent = 0.0d;
- double p = 0.0d;
- double base = 2.0;
- double totalTokens = (double)tokenCounts.getTotalTokens();
- for (MutableInt i : tokenCounts.getTokens().values()) {
- int termFreq = i.intValue();
+public class EvalExceptionUtils {
- p = (double) termFreq / totalTokens;
- ent += p * FastMath.log(base, p);
+ //these remove runtime info from the stacktraces so
+ //that actual causes can be counted.
+ private final static Pattern CAUSED_BY_SNIPPER =
+ Pattern.compile("(Caused by: [^:]+):[^\\r\\n]+");
+
+ public static String normalize(String stacktrace) {
+ if (StringUtils.isBlank(stacktrace)) {
+ return "";
}
- return ent;
+ String sortTrace = ExceptionUtils.trimMessage(stacktrace);
+
+ Matcher matcher = CAUSED_BY_SNIPPER.matcher(sortTrace);
+ sortTrace = matcher.replaceAll("$1");
+ return sortTrace.replaceAll("org.apache.tika.", "o.a.t.");
}
}
[tika] 03/05: TIKA-2926 -- add property for container exception in
RPW
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit dad1c722ceb7b8583f878d5e5bba88325f5f3530
Author: TALLISON <ta...@apache.org>
AuthorDate: Fri Aug 16 12:52:09 2019 -0400
TIKA-2926 -- add property for container exception in RPW
---
.../org/apache/tika/batch/RecursiveParserWrapperFSConsumerTest.java | 3 ++-
.../src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java | 2 +-
.../org/apache/tika/sax/AbstractRecursiveParserWrapperHandler.java | 5 ++++-
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/tika-batch/src/test/java/org/apache/tika/batch/RecursiveParserWrapperFSConsumerTest.java b/tika-batch/src/test/java/org/apache/tika/batch/RecursiveParserWrapperFSConsumerTest.java
index 443950f..028acf5 100644
--- a/tika-batch/src/test/java/org/apache/tika/batch/RecursiveParserWrapperFSConsumerTest.java
+++ b/tika-batch/src/test/java/org/apache/tika/batch/RecursiveParserWrapperFSConsumerTest.java
@@ -39,6 +39,7 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
import org.apache.tika.sax.BasicContentHandlerFactory;
+import org.apache.tika.sax.RecursiveParserWrapperHandler;
import org.junit.Test;
public class RecursiveParserWrapperFSConsumerTest extends TikaTest {
@@ -130,7 +131,7 @@ public class RecursiveParserWrapperFSConsumerTest extends TikaTest {
List<Metadata> results = JsonMetadataList.fromJson(new InputStreamReader(new ByteArrayInputStream(bytes), UTF_8));
assertEquals(2, results.size());
assertContains("another null pointer",
- results.get(0).get(TikaCoreProperties.TIKA_META_EXCEPTION_PREFIX + "runtime"));
+ results.get(0).get(RecursiveParserWrapperHandler.CONTAINER_EXCEPTION));
assertEquals("Nikolai Lobachevsky", results.get(0).get("author"));
assertEquals("embeddedAuthor", results.get(1).get("author"));
assertContains("some_embedded_content", results.get(1).get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT));
diff --git a/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java b/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java
index 4e05d74..37ed63c 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java
@@ -241,7 +241,7 @@ public class RecursiveParserWrapper extends ParserDecorator {
//try our best to record the problem in the metadata object
//then rethrow
String stackTrace = ExceptionUtils.getFilteredStackTrace(e);
- metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_PREFIX+"runtime", stackTrace);
+ metadata.add(RecursiveParserWrapperHandler.CONTAINER_EXCEPTION, stackTrace);
throw e;
} finally {
long elapsedMillis = System.currentTimeMillis() - started;
diff --git a/tika-core/src/main/java/org/apache/tika/sax/AbstractRecursiveParserWrapperHandler.java b/tika-core/src/main/java/org/apache/tika/sax/AbstractRecursiveParserWrapperHandler.java
index 8bdba65..c76080a 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/AbstractRecursiveParserWrapperHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/AbstractRecursiveParserWrapperHandler.java
@@ -46,8 +46,11 @@ public abstract class AbstractRecursiveParserWrapperHandler extends DefaultHandl
public final static Property EMBEDDED_RESOURCE_LIMIT_REACHED =
Property.internalBoolean(TikaCoreProperties.TIKA_META_EXCEPTION_PREFIX + "embedded_resource_limit_reached");
+ //exception in embedded file
public final static Property EMBEDDED_EXCEPTION = ParserUtils.EMBEDDED_EXCEPTION;
-
+ //exception in main file
+ public final static Property CONTAINER_EXCEPTION = Property.internalText(
+ TikaCoreProperties.TIKA_META_EXCEPTION_PREFIX+"runtime");
public final static Property EMBEDDED_RESOURCE_PATH =
Property.internalText(TikaCoreProperties.TIKA_META_PREFIX+"embedded_resource_path");
[tika] 05/05: improve logging in tika-server
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit e939504c6240e6e143b470c9eefe8615b3470fdd
Author: TALLISON <ta...@apache.org>
AuthorDate: Fri Aug 16 16:13:53 2019 -0400
improve logging in tika-server
---
.../src/main/java/org/apache/tika/server/ServerStatusWatcher.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/tika-server/src/main/java/org/apache/tika/server/ServerStatusWatcher.java b/tika-server/src/main/java/org/apache/tika/server/ServerStatusWatcher.java
index fa76d1e..b56ffc6 100644
--- a/tika-server/src/main/java/org/apache/tika/server/ServerStatusWatcher.java
+++ b/tika-server/src/main/java/org/apache/tika/server/ServerStatusWatcher.java
@@ -147,10 +147,14 @@ public class ServerStatusWatcher implements Runnable {
if (millisElapsed > serverTimeouts.getTaskTimeoutMillis()) {
serverStatus.setStatus(ServerStatus.STATUS.TIMEOUT);
if (status.fileName.isPresent()) {
- LOG.error("Timeout task {}, millis elapsed {}, file {}",
+ LOG.error("Timeout task {}, millis elapsed {}, file {}" +
+ "consider increasing the allowable time with the " +
+ "-taskTimeoutMillis flag",
status.task.toString(), Long.toString(millisElapsed), status.fileName.get());
} else {
- LOG.error("Timeout task {}, millis elapsed {}",
+ LOG.error("Timeout task {}, millis elapsed {}; " +
+ "consider increasing the allowable time with the " +
+ "-taskTimeoutMillis flag",
status.task.toString(), Long.toString(millisElapsed));
}
}