You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/06/14 20:49:05 UTC
[tika] branch branch_1x updated (ecbd316 -> 2512051)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a change to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git.
from ecbd316 undo idiocy
new 0b62157 avoid npe when lang code not found on classpath
new ebb22dd undo idiocy
new f309935 TIKA-2668 -- fix TaggedSAXException for Java 11-ea
new 2512051 TIKA-2660 -- enable building with Java 10
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
tika-bundle/pom.xml | 11 +-
.../test/java/org/apache/tika/bundle/BundleIT.java | 5 +-
tika-core/pom.xml | 1 +
.../apache/tika/detect/TrainedModelDetector.java | 3 +-
.../org/apache/tika/sax/TaggedSAXException.java | 1 -
.../apache/tika/sax/BodyContentHandlerTest.java | 34 ++-
tika-dl/pom.xml | 236 ++++++++++++++++++++-
tika-langdetect/pom.xml | 23 ++
tika-parent/pom.xml | 1 +
tika-parsers/pom.xml | 24 ++-
.../java/org/apache/tika/parser/mat/MatParser.java | 1 +
.../org/apache/tika/parser/rtf/TextExtractor.java | 19 +-
.../org/apache/tika/parser/mat/MatParserTest.java | 7 +-
.../apache/tika/parser/mbox/MboxParserTest.java | 16 +-
.../tika/parser/microsoft/JackcessParserTest.java | 3 +-
.../tika/parser/microsoft/OutlookParserTest.java | 4 +-
tika-server/pom.xml | 12 ++
17 files changed, 359 insertions(+), 42 deletions(-)
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 04/04: TIKA-2660 -- enable building with Java 10
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 251205158643e63e3a0a889a49b65093a9a26fe3
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Jun 14 16:41:06 2018 -0400
TIKA-2660 -- enable building with Java 10
---
tika-bundle/pom.xml | 11 +-
.../test/java/org/apache/tika/bundle/BundleIT.java | 5 +-
tika-core/pom.xml | 1 +
.../apache/tika/detect/TrainedModelDetector.java | 3 +-
tika-dl/pom.xml | 236 ++++++++++++++++++++-
tika-langdetect/pom.xml | 23 ++
tika-parent/pom.xml | 1 +
tika-parsers/pom.xml | 24 ++-
.../java/org/apache/tika/parser/mat/MatParser.java | 1 +
.../org/apache/tika/parser/rtf/TextExtractor.java | 19 +-
.../org/apache/tika/parser/mat/MatParserTest.java | 7 +-
.../apache/tika/parser/mbox/MboxParserTest.java | 16 +-
.../tika/parser/microsoft/JackcessParserTest.java | 3 +-
.../tika/parser/microsoft/OutlookParserTest.java | 4 +-
tika-server/pom.xml | 12 ++
15 files changed, 326 insertions(+), 40 deletions(-)
diff --git a/tika-bundle/pom.xml b/tika-bundle/pom.xml
index e749f00..b08cb9e 100644
--- a/tika-bundle/pom.xml
+++ b/tika-bundle/pom.xml
@@ -99,7 +99,7 @@
<dependency>
<groupId>org.apache.felix</groupId>
<artifactId>org.apache.felix.framework</artifactId>
- <version>5.6.4</version>
+ <version>5.6.10</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -111,7 +111,7 @@
<dependency>
<groupId>org.ops4j.pax.url</groupId>
<artifactId>pax-url-aether</artifactId>
- <version>2.5.2</version>
+ <version>2.5.4</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -246,11 +246,18 @@
javax.annotation;resolution:=optional,
javax.mail;resolution:=optional,
javax.mail.internet;resolution:=optional,
+ javax.net.ssl;resolution:=optional,
javax.servlet.annotation;resolution:=optional,
javax.servlet;resolution:=optional,
javax.servlet.http;resolution:=optional,
javax.measure.converter;resolution:=optional,
javax.ws.rs.core;resolution:=optional,
+ javax.xml.bind;resolution:=optional,
+ javax.xml.bind.annotation;resolution:=optional,
+ javax.xml.bind.annotation.adapters;resolution:=optional,
+ javax.xml.bind.attachment;resolution:=optional,
+ javax.xml.bind.helpers;resolution:=optional,
+ javax.xml.bind.util;resolution:=optional,
net.sf.ehcache;resolution:=optional,
nu.xom;resolution:=optional,
opendap.dap.http;resolution:=optional,
diff --git a/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java b/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
index 52eeddd..b544c56 100644
--- a/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
+++ b/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
@@ -33,6 +33,7 @@ import java.io.InputStream;
import java.io.StringWriter;
import java.io.Writer;
import java.net.URISyntaxException;
+import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
import java.util.jar.Attributes;
@@ -45,6 +46,7 @@ import org.apache.tika.Tika;
import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.fork.ForkParser;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.CompositeParser;
@@ -256,8 +258,7 @@ public class BundleIT {
ParseContext context = new ParseContext();
context.set(Parser.class, parser);
- try (InputStream stream =
- new FileInputStream("src/test/resources/test-documents.zip")) {
+ try (InputStream stream = TikaInputStream.get(Paths.get("src/test/resources/test-documents.zip"))) {
parser.parse(stream, handler, new Metadata(), context);
}
diff --git a/tika-core/pom.xml b/tika-core/pom.xml
index 5ca59f0..df168c1 100644
--- a/tika-core/pom.xml
+++ b/tika-core/pom.xml
@@ -105,6 +105,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>clirr-maven-plugin</artifactId>
+ <version>2.8</version>
<executions>
<execution>
<phase>verify</phase>
diff --git a/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java b/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
index dc8f142..ee19804 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/TrainedModelDetector.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
+import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
@@ -105,7 +106,7 @@ public abstract class TrainedModelDetector implements Detector {
float max = -1;
while (bytesRead != -1) {
- buf.flip(); // make buffer ready for read
+ ((Buffer)buf).flip(); // make buffer ready for read
while (buf.hasRemaining()) {
byte byt = buf.get();
diff --git a/tika-dl/pom.xml b/tika-dl/pom.xml
index 769683a..c66baed 100644
--- a/tika-dl/pom.xml
+++ b/tika-dl/pom.xml
@@ -36,8 +36,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <dl4j.version>0.8.0</dl4j.version>
- <dl4j.model.version>0.8.0-2</dl4j.model.version>
+ <dl4j.version>1.0.0-beta</dl4j.version>
+ <dl4j.model.version>0.9.1</dl4j.model.version>
</properties>
<dependencies>
@@ -51,9 +51,18 @@
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.9.9</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
@@ -62,7 +71,7 @@
<groupId>org.deeplearning4j</groupId>
<artifactId>deeplearning4j-keras</artifactId>
<version>${dl4j.model.version}</version>
- <!-- exclude this because of non-ASF friendly "do no evil" license.
+ <!-- exclude the first because of non-ASF friendly "do no evil" license.
Because this relies on tika-parsers, that should have ted-dunning's drop-in
-->
<exclusions>
@@ -75,10 +84,34 @@
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>jackson</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.deeplearning4j</groupId>
<artifactId>deeplearning4j-modelimport</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.deeplearning4j</groupId>
+ <artifactId>deeplearning4j-core</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
@@ -107,6 +140,19 @@
<artifactId>deeplearning4j-keras</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.deeplearning4j</groupId>
+ <artifactId>deeplearning4j-nn</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>jackson</artifactId>
+ </exclusion>
+
+ <exclusion>
<groupId>org.bytedeco</groupId>
<artifactId>javacpp</artifactId>
</exclusion>
@@ -114,14 +160,71 @@
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bytedeco.javacpp-presets</groupId>
+ <artifactId>hdf5-platform</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.deeplearning4j</groupId>
+ <artifactId>deeplearning4j-nn</artifactId>
+ <version>${dl4j.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>jackson</artifactId>
+ </exclusion>
+
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.16.20</version>
+ </dependency>
+ <dependency>
<groupId>org.datavec</groupId>
<artifactId>datavec-data-image</artifactId>
<version>${dl4j.version}</version>
<exclusions>
<exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
@@ -130,6 +233,27 @@
<artifactId>javacpp</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>jackson</artifactId>
+ </exclusion>
+
+ <exclusion>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacv</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</exclusion>
@@ -141,6 +265,96 @@
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bytedeco.javacpp-presets</groupId>
+ <artifactId>opencv-platform</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacv</artifactId>
+ <version>1.4.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.bytedeco.javacpp-presets</groupId>
+ <artifactId>opencv</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.bytedeco.javacpp-presets</groupId>
+ <artifactId>opencv-platform</artifactId>
+ <version>3.4.1-1.4.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.bytedeco.javacpp-presets</groupId>
+ <artifactId>opencv</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.bytedeco.javacpp-presets</groupId>
+ <artifactId>opencv</artifactId>
+ <version>3.4.1-1.4.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-common</artifactId>
+ <version>1.0.0-beta</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>jackson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-jackson</artifactId>
+ <version>1.0.0-beta</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-api</artifactId>
+ <version>1.0.0-beta</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -152,17 +366,21 @@
<groupId>org.bytedeco</groupId>
<artifactId>javacpp</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-native</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>org.bytedeco</groupId>
- <artifactId>javacpp</artifactId>
- <version>1.3.2</version>
+ <groupId>org.nd4j</groupId>
+ <artifactId>nd4j-native</artifactId>
+ <version>${dl4j.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- <version>${commons.compress.version}</version>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ <version>1.4.1</version>
</dependency>
</dependencies>
diff --git a/tika-langdetect/pom.xml b/tika-langdetect/pom.xml
index fb30934..b879a7b 100644
--- a/tika-langdetect/pom.xml
+++ b/tika-langdetect/pom.xml
@@ -40,6 +40,29 @@
<artifactId>tika-core</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- for java 10 -->
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>${jaxb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-core</artifactId>
+ <version>${jaxb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>${jaxb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+
<dependency>
<groupId>com.optimaize.languagedetector</groupId>
<artifactId>language-detector</artifactId>
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 02bd802..3a37a42 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -312,6 +312,7 @@
<cxf.version>3.0.16</cxf.version>
<slf4j.version>1.7.24</slf4j.version>
<jackson.version>2.9.5</jackson.version>
+ <jaxb.version>2.3.0</jaxb.version>
</properties>
<build>
diff --git a/tika-parsers/pom.xml b/tika-parsers/pom.xml
index 885c3b9..08513e4 100644
--- a/tika-parsers/pom.xml
+++ b/tika-parsers/pom.xml
@@ -78,6 +78,28 @@
<scope>test</scope>
</dependency>
+ <!-- for java 10 -->
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>${jaxb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-core</artifactId>
+ <version>${jaxb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.xml.bind</groupId>
+ <artifactId>jaxb-impl</artifactId>
+ <version>${jaxb.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+
<!-- Externally Maintained Parsers -->
<dependency>
<groupId>org.gagravarr</groupId>
@@ -130,7 +152,7 @@
<dependency>
<groupId>org.tallison</groupId>
<artifactId>jmatio</artifactId>
- <version>1.2</version>
+ <version>1.3</version>
</dependency>
<dependency>
<groupId>org.apache.james</groupId>
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/mat/MatParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/mat/MatParser.java
index 3fbefb0..2db8bef 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/mat/MatParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/mat/MatParser.java
@@ -74,6 +74,7 @@ public class MatParser extends AbstractParser {
//Extract information from header file
MatFileReader mfr = new MatFileReader(tis.getFile()); //input .mat file
+
MatFileHeader hdr = mfr.getMatFileHeader(); //.mat header information
// Example header: "MATLAB 5.0 MAT-file, Platform: MACI64, Created on: Sun Mar 2 23:41:57 2014"
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java b/tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java
index a0dac99..1eb7220 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/rtf/TextExtractor.java
@@ -20,6 +20,7 @@ package org.apache.tika.parser.rtf;
import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;
+import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
@@ -238,7 +239,7 @@ final class TextExtractor {
// Used when we decode bytes -> chars using CharsetDecoder:
private final char[] outputArray = new char[128];
- private final CharBuffer outputBuffer = CharBuffer.wrap(outputArray);
+ private final Buffer outputCharBuffer = CharBuffer.wrap(outputArray);
// Holds the font table from this RTF doc, mapping
// the font number (from \fN control word) to the
// corresponding charset:
@@ -262,7 +263,7 @@ final class TextExtractor {
// for text output:
private byte[] pendingBytes = new byte[16];
private int pendingByteCount;
- private ByteBuffer pendingByteBuffer = ByteBuffer.wrap(pendingBytes);
+ private Buffer pendingByteBuffer = ByteBuffer.wrap(pendingBytes);
// Holds pending chars for text output
private char[] pendingChars = new char[10];
private int pendingCharCount;
@@ -663,16 +664,16 @@ final class TextExtractor {
final CharsetDecoder decoder = getDecoder();
pendingByteBuffer.limit(pendingByteCount);
assert pendingByteBuffer.position() == 0;
- assert outputBuffer.position() == 0;
+ assert outputCharBuffer.position() == 0;
while (true) {
// We pass true for endOfInput because, when
// we are called, we should have seen a
// complete sequence of characters for this
// charset:
- final CoderResult result = decoder.decode(pendingByteBuffer, outputBuffer, true);
+ final CoderResult result = decoder.decode((ByteBuffer)pendingByteBuffer, (CharBuffer) outputCharBuffer, true);
- final int pos = outputBuffer.position();
+ final int pos = outputCharBuffer.position();
if (pos > 0) {
if (inHeader || fieldState == 1) {
pendingBuffer.append(outputArray, 0, pos);
@@ -680,7 +681,7 @@ final class TextExtractor {
lazyStartParagraph();
out.characters(outputArray, 0, pos);
}
- outputBuffer.position(0);
+ outputCharBuffer.position(0);
}
if (result == CoderResult.UNDERFLOW) {
@@ -689,9 +690,9 @@ final class TextExtractor {
}
while (true) {
- final CoderResult result = decoder.flush(outputBuffer);
+ final CoderResult result = decoder.flush((CharBuffer) outputCharBuffer);
- final int pos = outputBuffer.position();
+ final int pos = outputCharBuffer.position();
if (pos > 0) {
if (inHeader || fieldState == 1) {
pendingBuffer.append(outputArray, 0, pos);
@@ -699,7 +700,7 @@ final class TextExtractor {
lazyStartParagraph();
out.characters(outputArray, 0, pos);
}
- outputBuffer.position(0);
+ outputCharBuffer.position(0);
}
if (result == CoderResult.UNDERFLOW) {
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mat/MatParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
index 0b31fea..b87b2b6 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/mat/MatParserTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import java.io.InputStream;
+import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
@@ -31,7 +32,7 @@ import org.junit.Test;
/**
* Test cases to exercise the {@link MatParser}.
*/
-public class MatParserTest {
+public class MatParserTest extends TikaTest {
@Test
public void testParser() throws Exception {
AutoDetectParser parser = new AutoDetectParser();
@@ -39,7 +40,7 @@ public class MatParserTest {
Metadata metadata = new Metadata();
String path = "/test-documents/breidamerkurjokull_radar_profiles_2009.mat";
- try (InputStream stream = MatParser.class.getResourceAsStream(path)) {
+ try (InputStream stream = getResourceAsStream(path)) {
parser.parse(stream, handler, metadata, new ParseContext());
}
@@ -69,7 +70,7 @@ public class MatParserTest {
Metadata metadata = new Metadata();
String path = "/test-documents/test_mat_text.mat";
- try (InputStream stream = MatParser.class.getResourceAsStream(path)) {
+ try (InputStream stream = getResourceAsStream(path)) {
parser.parse(stream, handler, metadata, new ParseContext());
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java
index 043261a..6b85968 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/mbox/MboxParserTest.java
@@ -43,10 +43,6 @@ public class MboxParserTest extends TikaTest {
private TypeDetector typeDetector;
private MboxParser mboxParser;
- private static InputStream getStream(String name) {
- return MboxParserTest.class.getClass().getResourceAsStream(name);
- }
-
@Before
public void setUp() throws Exception {
typeDetector = new TypeDetector();
@@ -63,7 +59,7 @@ public class MboxParserTest extends TikaTest {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
- try (InputStream stream = getStream("/test-documents/simple.mbox")) {
+ try (InputStream stream = getResourceAsStream("/test-documents/simple.mbox")) {
mboxParser.parse(stream, handler, metadata, recursingContext);
}
@@ -89,7 +85,7 @@ public class MboxParserTest extends TikaTest {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
- try (InputStream stream = getStream("/test-documents/headers.mbox")) {
+ try (InputStream stream = getResourceAsStream("/test-documents/headers.mbox")) {
mboxParser.parse(stream, handler, metadata, recursingContext);
}
@@ -112,7 +108,7 @@ public class MboxParserTest extends TikaTest {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
- try (InputStream stream = getStream("/test-documents/multiline.mbox")) {
+ try (InputStream stream = getResourceAsStream("/test-documents/multiline.mbox")) {
mboxParser.parse(stream, handler, metadata, recursingContext);
}
@@ -127,7 +123,7 @@ public class MboxParserTest extends TikaTest {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
- try (InputStream stream = getStream("/test-documents/quoted.mbox")) {
+ try (InputStream stream = getResourceAsStream("/test-documents/quoted.mbox")) {
mboxParser.parse(stream, handler, metadata, recursingContext);
}
@@ -140,7 +136,7 @@ public class MboxParserTest extends TikaTest {
ContentHandler handler = new BodyContentHandler();
Metadata metadata = new Metadata();
- try (InputStream stream = getStream("/test-documents/complex.mbox")) {
+ try (InputStream stream = getResourceAsStream("/test-documents/complex.mbox")) {
mboxParser.parse(stream, handler, metadata, recursingContext);
}
@@ -163,7 +159,7 @@ public class MboxParserTest extends TikaTest {
ParseContext context = new ParseContext();
context.set(Parser.class, new AutoDetectParser());
- try (InputStream stream = getStream("/test-documents/single_mail.mbox")) {
+ try (InputStream stream = getResourceAsStream("/test-documents/single_mail.mbox")) {
mboxParser.parse(stream, handler, metadata, context);
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/JackcessParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/JackcessParserTest.java
index e6aef53..f39b961 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/JackcessParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/JackcessParserTest.java
@@ -75,7 +75,8 @@ public class JackcessParserTest extends TikaTest {
assertContains("<th>ShortTextField</th>", mainContent);
//test date format
- assertContains("6/24/15", mainContent);
+ //java 8 is 6/24/15 ...java 10 is 2015-06-24
+ assertTrue (mainContent.contains("6/24/15") || mainContent.contains("2015-06-24"));
//test that markup is stripped
assertContains("over the bold italic dog", mainContent);
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
index acfb754..9e3b358 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
@@ -271,10 +271,10 @@ public class OutlookParserTest extends TikaTest {
// As the HTML version should have been processed, ensure
// we got some of the links
- String content = sw.toString().replaceAll("<p>\\s+", "<p>");
+ String content = sw.toString().replaceAll("[\\r\\n\\t]+", " ").replaceAll(" +", " ");
assertContains("<dd>New Outlook User</dd>", content);
assertContains("designed <i>to help you", content);
- assertContains("<p><a href=\"http://r.office.microsoft.com/r/rlidOutlookWelcomeMail10?clid=1033\">Cached Exchange Mode</a>", content);
+ assertContains("<p> <a href=\"http://r.office.microsoft.com/r/rlidOutlookWelcomeMail10?clid=1033\">Cached Exchange Mode</a>", content);
// Link - check text around it, and the link itself
assertContains("sign up for a free subscription", content);
diff --git a/tika-server/pom.xml b/tika-server/pom.xml
index 850e3f2..814375c 100644
--- a/tika-server/pom.xml
+++ b/tika-server/pom.xml
@@ -54,6 +54,12 @@
<groupId>${project.groupId}</groupId>
<artifactId>tika-langdetect</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
@@ -95,6 +101,12 @@
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.4.4</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 02/04: undo idiocy
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit ebb22ddcc77837effd8d9616e251a565fd43c802
Author: tballison <ta...@mitre.org>
AuthorDate: Tue Jun 12 09:38:50 2018 -0400
undo idiocy
---
.../main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
index a5bd96b..1fc2ca3 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
@@ -58,7 +58,7 @@ public class CommonTokenCountManager {
Set<String> set = commonTokenMap.get(defaultLangCode);
if (set == null) {
LOG.warn("No common tokens for default language: '"+defaultLangCode+"'");
- commonTokenMap.put(defaultLangCode, new HashSet<>());
+ commonTokenMap.put(defaultLangCode, new HashSet<String>());
}
}
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 01/04: avoid npe when lang code not found on classpath
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 0b62157b090138c857d03ffb1d806a1a96a89fae
Author: tballison <ta...@mitre.org>
AuthorDate: Tue Jun 12 08:55:19 2018 -0400
avoid npe when lang code not found on classpath
---
.../main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
index 1fc2ca3..a5bd96b 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/tokens/CommonTokenCountManager.java
@@ -58,7 +58,7 @@ public class CommonTokenCountManager {
Set<String> set = commonTokenMap.get(defaultLangCode);
if (set == null) {
LOG.warn("No common tokens for default language: '"+defaultLangCode+"'");
- commonTokenMap.put(defaultLangCode, new HashSet<String>());
+ commonTokenMap.put(defaultLangCode, new HashSet<>());
}
}
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.
[tika] 03/04: TIKA-2668 -- fix TaggedSAXException for Java 11-ea
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit f3099356527805426135987cb62353c9da24b573
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Jun 14 12:45:20 2018 -0400
TIKA-2668 -- fix TaggedSAXException for Java 11-ea
---
.../org/apache/tika/sax/TaggedSAXException.java | 1 -
.../apache/tika/sax/BodyContentHandlerTest.java | 34 +++++++++++++++++++++-
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java b/tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java
index d2db72a..19b4ea8 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/TaggedSAXException.java
@@ -38,7 +38,6 @@ public class TaggedSAXException extends SAXException {
*/
public TaggedSAXException(SAXException original, Object tag) {
super(original.getMessage(), original);
- initCause(original); // SAXException has it's own chaining mechanism!
this.tag = tag;
}
diff --git a/tika-core/src/test/java/org/apache/tika/sax/BodyContentHandlerTest.java b/tika-core/src/test/java/org/apache/tika/sax/BodyContentHandlerTest.java
index bf42706..c4acb03 100644
--- a/tika-core/src/test/java/org/apache/tika/sax/BodyContentHandlerTest.java
+++ b/tika-core/src/test/java/org/apache/tika/sax/BodyContentHandlerTest.java
@@ -20,15 +20,21 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.mock.MockParser;
import org.junit.Test;
/**
* Test cases for the {@link BodyContentHandler} class.
*/
-public class BodyContentHandlerTest {
+public class BodyContentHandlerTest extends TikaTest {
/**
* Test that the conversion to an {@link OutputStream} doesn't leave
@@ -49,4 +55,30 @@ public class BodyContentHandlerTest {
assertEquals("Test text\n", buffer.toString(UTF_8.name()));
}
+ @Test
+ public void testLimit() throws Exception {
+ //TIKA-2668 - java 11-ea
+ Parser p = new MockParser();
+ WriteOutContentHandler handler = new WriteOutContentHandler(15);
+ Metadata metadata = new Metadata();
+ ParseContext parseContext = new ParseContext();
+ Parser[] parsers = new Parser[1];
+ parsers[0] = p;
+ Parser autoDetectParser = new AutoDetectParser(parsers);
+ try (InputStream is = getResourceAsStream("/test-documents/example.xml")) {
+ autoDetectParser.parse(is, handler, metadata, parseContext);
+ } catch (Exception e) {
+ tryToFindIllegalStateException(e);
+ }
+ assertEquals("hello wo", handler.toString().trim());
+ }
+
+ private void tryToFindIllegalStateException(Throwable e) throws Exception {
+ if (e instanceof IllegalStateException) {
+ throw (Exception)e;
+ }
+ if (e.getCause() != null) {
+ tryToFindIllegalStateException(e.getCause());
+ }
+ }
}
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.