You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/02/27 18:29:47 UTC
tika git commit: TIKA-2273 -- fix configuration of encoding detectors
when parsers are loaded statically.
Repository: tika
Updated Branches:
refs/heads/master 4a4e89a0c -> e7a0c3eec
TIKA-2273 -- fix configuration of encoding detectors when parsers are loaded statically.
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/e7a0c3ee
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/e7a0c3ee
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/e7a0c3ee
Branch: refs/heads/master
Commit: e7a0c3eece98ac81b3813aeb429b24f16090201c
Parents: 4a4e89a
Author: tballison <ta...@mitre.org>
Authored: Mon Feb 27 13:29:39 2017 -0500
Committer: tballison <ta...@mitre.org>
Committed: Mon Feb 27 13:29:39 2017 -0500
----------------------------------------------------------------------
.../java/org/apache/tika/bundle/BundleIT.java | 3 -
.../org/apache/tika/config/ServiceLoader.java | 4 +-
.../java/org/apache/tika/config/TikaConfig.java | 34 ++++-
.../detect/NonDetectingEncodingDetector.java | 12 +-
.../parser/AbstractEncodingDetectorParser.java | 23 ++--
.../apache/tika/parser/AutoDetectParser.java | 2 -
.../org/apache/tika/parser/DefaultParser.java | 23 +++-
.../tika/parser/code/SourceCodeParser.java | 9 ++
.../org/apache/tika/parser/html/HtmlParser.java | 9 ++
.../org/apache/tika/parser/txt/TXTParser.java | 9 ++
.../tika/config/TikaEncodingDetectorTest.java | 131 ++++++++++++++++++-
.../apache/tika/parser/txt/TXTParserTest.java | 29 ----
...73-encoding-detector-outside-static-init.xml | 34 +++++
...KA-2273-non-detecting-params-bad-charset.xml | 29 ++++
.../config/TIKA-2273-non-detecting-params.xml | 29 ++++
15 files changed, 314 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
----------------------------------------------------------------------
diff --git a/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java b/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
index 299a540..c610e9a 100644
--- a/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
+++ b/tika-bundle/src/test/java/org/apache/tika/bundle/BundleIT.java
@@ -52,7 +52,6 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.parser.internal.Activator;
import org.apache.tika.parser.ocr.TesseractOCRParser;
import org.apache.tika.sax.BodyContentHandler;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.ops4j.pax.exam.Configuration;
@@ -140,7 +139,6 @@ public class BundleIT {
@Test
- @Ignore("temporarily")
public void testForkParser() throws Exception {
ForkParser parser = new ForkParser(Activator.class.getClassLoader(), defaultParser);
String data = "<!DOCTYPE html>\n<html><body><p>test <span>content</span></p></body></html>";
@@ -251,7 +249,6 @@ public class BundleIT {
@Test
- @Ignore("temporarily")
public void testTikaBundle() throws Exception {
Tika tika = new Tika();
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-core/src/main/java/org/apache/tika/config/ServiceLoader.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/config/ServiceLoader.java b/tika-core/src/main/java/org/apache/tika/config/ServiceLoader.java
index f98117e..66c2883 100644
--- a/tika-core/src/main/java/org/apache/tika/config/ServiceLoader.java
+++ b/tika-core/src/main/java/org/apache/tika/config/ServiceLoader.java
@@ -16,6 +16,8 @@
*/
package org.apache.tika.config;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
@@ -30,8 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
/**
* Internal utility class that Tika uses to look up service providers.
*
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
index f6e58af..06eb43b 100644
--- a/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
@@ -55,6 +55,7 @@ import org.apache.tika.mime.MediaTypeRegistry;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.mime.MimeTypesFactory;
+import org.apache.tika.parser.AbstractEncodingDetectorParser;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.DefaultParser;
@@ -586,11 +587,11 @@ public class TikaConfig {
// Default constructor fallback
if (loaded == null) {
- loaded = loadedClass.newInstance();
+ loaded = newInstance(loadedClass);
}
} else {
// Regular class, create as-is
- loaded = loadedClass.newInstance();
+ loaded = newInstance(loadedClass);
// TODO Support arguments, needed for Translators etc
// See the thread "Configuring parsers and translators" for details
}
@@ -623,9 +624,18 @@ public class TikaConfig {
} catch (InstantiationException e) {
throw new TikaException(
"Unable to instantiate a "+getLoaderTagName()+" class: " + name, e);
+ } catch (NoSuchMethodException e) {
+ throw new TikaException(
+ "Unable to find the right constructor for "+getLoaderTagName()+" class: " + name, e);
}
}
+ T newInstance(Class<? extends T> loadedClass) throws
+ IllegalAccessException, InstantiationException,
+ NoSuchMethodException, InvocationTargetException {
+ return loadedClass.newInstance();
+ }
+
/**
* Gets parameters from a given
* @param el xml node which has {@link #PARAMS_TAG_NAME} child
@@ -718,6 +728,14 @@ public class TikaConfig {
// Try the possible default and composite parser constructors
if (parser == null) {
try {
+ c = parserClass.getConstructor(MediaTypeRegistry.class,
+ ServiceLoader.class, Collection.class, EncodingDetector.class);
+ parser = c.newInstance(registry, loader, excludeParsers, encodingDetector);
+ }
+ catch (NoSuchMethodException me) {}
+ }
+ if (parser == null) {
+ try {
c = parserClass.getConstructor(MediaTypeRegistry.class, ServiceLoader.class, Collection.class);
parser = c.newInstance(registry, loader, excludeParsers);
}
@@ -752,6 +770,17 @@ public class TikaConfig {
}
return parser;
}
+
+ @Override
+ Parser newInstance(Class<? extends Parser> loadedClass) throws IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException {
+ if (AbstractEncodingDetectorParser.class.isAssignableFrom(loadedClass)) {
+ Constructor ctor = loadedClass.getConstructor(EncodingDetector.class);
+ return (Parser) ctor.newInstance(encodingDetector);
+ } else {
+ return loadedClass.newInstance();
+ }
+ }
+
@Override
Parser decorate(Parser created, Element element) throws IOException, TikaException {
Parser parser = created;
@@ -770,6 +799,7 @@ public class TikaConfig {
// All done with decoration
return parser;
}
+
}
private static class DetectorXmlLoader extends XmlLoader<CompositeDetector,Detector> {
boolean supportsComposite() { return true; }
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-core/src/main/java/org/apache/tika/detect/NonDetectingEncodingDetector.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/detect/NonDetectingEncodingDetector.java b/tika-core/src/main/java/org/apache/tika/detect/NonDetectingEncodingDetector.java
index 2951033..ce95907 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/NonDetectingEncodingDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/NonDetectingEncodingDetector.java
@@ -22,10 +22,11 @@ import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import org.apache.tika.config.Field;
import org.apache.tika.metadata.Metadata;
/**
- * Always returns the charset passed in via the initializer.
+ * Always returns the charset passed in via the initializer
*/
public class NonDetectingEncodingDetector implements EncodingDetector {
//would have preferred final, but need mutability for
@@ -48,7 +49,12 @@ public class NonDetectingEncodingDetector implements EncodingDetector {
return charset;
}
- public void setCharset(Charset charset) {
- this.charset = charset;
+ @Field
+ private void setCharset(String charsetName) {
+ this.charset = Charset.forName(charsetName);
+ }
+
+ public Charset getCharset() {
+ return charset;
}
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java b/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
index 2db5e75..9185004 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/AbstractEncodingDetectorParser.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser;
import org.apache.tika.config.TikaConfig;
+import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
@@ -26,14 +27,18 @@ import org.apache.tika.detect.EncodingDetector;
*/
public abstract class AbstractEncodingDetectorParser extends AbstractParser {
- //In the most common scenario, this will be set
- //at the creation via AutoDetectParser and TikaConfig
- //If it isn't set then, lazily initialize it in the
- //getter.
+
private EncodingDetector encodingDetector;
private static final Object lock = new Object();
+ public AbstractEncodingDetectorParser() {
+ encodingDetector = new DefaultEncodingDetector();
+ }
+
+ public AbstractEncodingDetectorParser(EncodingDetector encodingDetector) {
+ this.encodingDetector = encodingDetector;
+ }
/**
* Look for an EncodingDetetor in the ParseContext. If it hasn't been
* passed in, use the original EncodingDetector from initialization.
@@ -52,16 +57,6 @@ public abstract class AbstractEncodingDetectorParser extends AbstractParser {
}
public EncodingDetector getEncodingDetector() {
- //lazily initialize to the default if one hasn't been set
- //via TikaConfig loading
- if (encodingDetector == null) {
- synchronized (lock) {
- //check again
- if (encodingDetector == null) {
- encodingDetector = TikaConfig.getDefaultConfig().getEncodingDetector();
- }
- }
- }
return encodingDetector;
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
index f1a7422..fe1c659 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/AutoDetectParser.java
@@ -74,13 +74,11 @@ public class AutoDetectParser extends CompositeParser {
public AutoDetectParser(Detector detector, Parser...parsers) {
super(MediaTypeRegistry.getDefaultRegistry(), parsers);
setDetector(detector);
- //config = TikaConfig.getDefaultConfig();
}
public AutoDetectParser(TikaConfig config) {
super(config.getMediaTypeRegistry(), config.getParser());
setDetector(config.getDetector());
- //this.config = config;
}
/**
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-core/src/main/java/org/apache/tika/parser/DefaultParser.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/parser/DefaultParser.java b/tika-core/src/main/java/org/apache/tika/parser/DefaultParser.java
index b46f7c5..233b2a0 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/DefaultParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/DefaultParser.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.tika.config.ServiceLoader;
+import org.apache.tika.detect.DefaultEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MediaTypeRegistry;
@@ -55,15 +56,27 @@ public class DefaultParser extends CompositeParser {
if (encodingDetector != null) {
for (Parser p : parsers) {
- if (p instanceof AbstractEncodingDetectorParser) {
- ((AbstractEncodingDetectorParser)p).setEncodingDetector(encodingDetector);
- }
+ setEncodingDetector(p, encodingDetector);
}
}
ServiceLoaderUtils.sortLoadedClasses(parsers);
return parsers;
}
+ //recursively go through the parsers and set the encoding detector
+ //as configured in the config file
+ private static void setEncodingDetector(Parser p, EncodingDetector encodingDetector) {
+ if (p instanceof AbstractEncodingDetectorParser) {
+ ((AbstractEncodingDetectorParser)p).setEncodingDetector(encodingDetector);
+ } else if (p instanceof CompositeParser) {
+ for (Parser child : ((CompositeParser)p).getAllComponentParsers()) {
+ setEncodingDetector(child, encodingDetector);
+ }
+ } else if (p instanceof ParserDecorator) {
+ setEncodingDetector(((ParserDecorator)p).getWrappedParser(), encodingDetector);
+ }
+ }
+
private transient final ServiceLoader loader;
public DefaultParser(MediaTypeRegistry registry, ServiceLoader loader,
@@ -75,7 +88,7 @@ public class DefaultParser extends CompositeParser {
public DefaultParser(MediaTypeRegistry registry, ServiceLoader loader,
Collection<Class<? extends Parser>> excludeParsers) {
- super(registry, getDefaultParsers(loader, null), excludeParsers);
+ super(registry, getDefaultParsers(loader, new DefaultEncodingDetector(loader)), excludeParsers);
this.loader = loader;
}
@@ -84,7 +97,7 @@ public class DefaultParser extends CompositeParser {
}
public DefaultParser(MediaTypeRegistry registry, ServiceLoader loader) {
- this(registry, loader, null, null);
+ this(registry, loader, null, new DefaultEncodingDetector(loader));
}
public DefaultParser(MediaTypeRegistry registry, ClassLoader loader) {
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/main/java/org/apache/tika/parser/code/SourceCodeParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/code/SourceCodeParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/code/SourceCodeParser.java
index 0047034..bfeb89f 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/code/SourceCodeParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/code/SourceCodeParser.java
@@ -34,6 +34,7 @@ import com.uwyn.jhighlight.renderer.Renderer;
import com.uwyn.jhighlight.renderer.XhtmlRendererFactory;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -72,6 +73,14 @@ public class SourceCodeParser extends AbstractEncodingDetectorParser {
//Parse the HTML document
private static final Schema HTML_SCHEMA = new HTMLSchema();
+ public SourceCodeParser() {
+ super();
+ }
+
+ public SourceCodeParser(EncodingDetector encodingDetector) {
+ super(encodingDetector);
+ }
+
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
return TYPES_TO_RENDERER.keySet();
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
index a2fac98..1538111 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -69,6 +70,14 @@ public class HtmlParser extends AbstractEncodingDetectorParser {
return SUPPORTED_TYPES;
}
+ public HtmlParser() {
+ super();
+ }
+
+ public HtmlParser(EncodingDetector encodingDetector) {
+ super(encodingDetector);
+ }
+
public void parse(
InputStream stream, ContentHandler handler,
Metadata metadata, ParseContext context)
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/main/java/org/apache/tika/parser/txt/TXTParser.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/TXTParser.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/TXTParser.java
index 382516f..15425d5 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/TXTParser.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/TXTParser.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -60,6 +61,14 @@ public class TXTParser extends AbstractEncodingDetectorParser {
return SUPPORTED_TYPES;
}
+ public TXTParser() {
+ super();
+ }
+
+ public TXTParser(EncodingDetector encodingDetector) {
+ super(encodingDetector);
+ }
+
public void parse(
InputStream stream, ContentHandler handler,
Metadata metadata, ParseContext context)
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index af4b5d5..f23eab4 100644
--- a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -19,14 +19,27 @@ package org.apache.tika.config;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.tika.Tika;
import org.apache.tika.detect.CompositeEncodingDetector;
-import org.apache.tika.detect.NonDetectingEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.detect.NonDetectingEncodingDetector;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AbstractEncodingDetectorParser;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.CompositeParser;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.ParserDecorator;
import org.apache.tika.parser.html.HtmlEncodingDetector;
import org.apache.tika.parser.txt.Icu4jEncodingDetector;
+import org.apache.tika.parser.txt.TXTParser;
import org.apache.tika.parser.txt.UniversalEncodingDetector;
import org.junit.Test;
@@ -36,7 +49,7 @@ public class TikaEncodingDetectorTest extends AbstractTikaConfigTest {
public void testDefault() {
EncodingDetector detector = TikaConfig.getDefaultConfig().getEncodingDetector();
assertTrue(detector instanceof CompositeEncodingDetector);
- List<EncodingDetector> detectors = ((CompositeEncodingDetector)detector).getDetectors();
+ List<EncodingDetector> detectors = ((CompositeEncodingDetector) detector).getDetectors();
assertEquals(3, detectors.size());
assertTrue(detectors.get(0) instanceof HtmlEncodingDetector);
assertTrue(detectors.get(1) instanceof UniversalEncodingDetector);
@@ -48,12 +61,12 @@ public class TikaEncodingDetectorTest extends AbstractTikaConfigTest {
TikaConfig config = getConfig("TIKA-2273-blacklist-encoding-detector-default.xml");
EncodingDetector detector = config.getEncodingDetector();
assertTrue(detector instanceof CompositeEncodingDetector);
- List<EncodingDetector> detectors = ((CompositeEncodingDetector)detector).getDetectors();
+ List<EncodingDetector> detectors = ((CompositeEncodingDetector) detector).getDetectors();
assertEquals(2, detectors.size());
EncodingDetector detector1 = detectors.get(0);
assertTrue(detector1 instanceof CompositeEncodingDetector);
- List<EncodingDetector> detectors1Children = ((CompositeEncodingDetector)detector1).getDetectors();
+ List<EncodingDetector> detectors1Children = ((CompositeEncodingDetector) detector1).getDetectors();
assertEquals(2, detectors1Children.size());
assertTrue(detectors1Children.get(0) instanceof UniversalEncodingDetector);
assertTrue(detectors1Children.get(1) instanceof Icu4jEncodingDetector);
@@ -67,10 +80,116 @@ public class TikaEncodingDetectorTest extends AbstractTikaConfigTest {
TikaConfig config = getConfig("TIKA-2273-parameterize-encoding-detector.xml");
EncodingDetector detector = config.getEncodingDetector();
assertTrue(detector instanceof CompositeEncodingDetector);
- List<EncodingDetector> detectors = ((CompositeEncodingDetector)detector).getDetectors();
+ List<EncodingDetector> detectors = ((CompositeEncodingDetector) detector).getDetectors();
assertEquals(2, detectors.size());
- assertTrue(((Icu4jEncodingDetector)detectors.get(0)).getStripMarkup());
+ assertTrue(((Icu4jEncodingDetector) detectors.get(0)).getStripMarkup());
assertTrue(detectors.get(1) instanceof NonDetectingEncodingDetector);
+ }
+
+ @Test
+ public void testEncodingDetectorsAreLoaded() {
+ EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser) new TXTParser()).getEncodingDetector();
+
+ assertTrue(encodingDetector instanceof CompositeEncodingDetector);
+ }
+
+ @Test
+ public void testEncodingDetectorConfigurability() throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(
+ getResourceAsStream("/org/apache/tika/config/TIKA-2273-no-icu4j-encoding-detector.xml"));
+ AutoDetectParser p = new AutoDetectParser(tikaConfig);
+
+ try {
+ Metadata metadata = getXML("english.cp500.txt", p).metadata;
+ fail("can't detect w/out ICU");
+ } catch (TikaException e) {
+ assertContains("Failed to detect", e.getMessage());
+ }
+
+ Tika tika = new Tika(tikaConfig);
+ try {
+ String txt = tika.parseToString(
+ getResourceAsFile("/test-documents/english.cp500.txt"));
+ fail("can't detect w/out ICU");
+ } catch (TikaException e) {
+ assertContains("Failed to detect", e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void testNonDetectingDetectorParams() throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(
+ getResourceAsStream("/org/apache/tika/config/TIKA-2273-non-detecting-params.xml"));
+ AutoDetectParser p = new AutoDetectParser(tikaConfig);
+ List<Parser> parsers = new ArrayList<>();
+ findEncodingDetectionParsers(p, parsers);
+
+ assertEquals(3, parsers.size());
+ EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser)parsers.get(0)).getEncodingDetector();
+ assertTrue(encodingDetector instanceof CompositeEncodingDetector);
+ assertEquals(1, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
+ EncodingDetector child = ((CompositeEncodingDetector) encodingDetector).getDetectors().get(0);
+ assertTrue( child instanceof NonDetectingEncodingDetector);
+
+ assertEquals(StandardCharsets.UTF_16LE, ((NonDetectingEncodingDetector)child).getCharset());
+
+ }
+
+ @Test
+ public void testNonDetectingDetectorParamsBadCharset() throws Exception {
+ try {
+ TikaConfig tikaConfig = new TikaConfig(
+ getResourceAsStream("/org/apache/tika/config/TIKA-2273-non-detecting-params-bad-charset.xml"));
+ fail("should have thrown TikaConfigException");
+ } catch (TikaConfigException e) {
+
+ }
+ }
+
+ @Test
+ public void testConfigurabilityOfUserSpecified() throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(
+ getResourceAsStream("/org/apache/tika/config/TIKA-2273-encoding-detector-outside-static-init.xml"));
+ AutoDetectParser p = new AutoDetectParser(tikaConfig);
+
+ //make sure that all static and non-static parsers are using the same encoding detector!
+ List<Parser> parsers = new ArrayList<>();
+ findEncodingDetectionParsers(p, parsers);
+
+ assertEquals(3, parsers.size());
+
+ for (Parser encodingDetectingParser : parsers) {
+ EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser) encodingDetectingParser).getEncodingDetector();
+ assertTrue(encodingDetector instanceof CompositeEncodingDetector);
+ assertEquals(2, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
+ for (EncodingDetector child : ((CompositeEncodingDetector) encodingDetector).getDetectors()) {
+ assertNotContained("cu4j", child.getClass().getCanonicalName());
+ }
+ }
+
+ //also just make sure this is still true
+ try {
+ Metadata metadata = getXML("english.cp500.txt", p).metadata;
+ fail("can't detect w/out ICU");
+ } catch (TikaException e) {
+ assertContains("Failed to detect", e.getMessage());
+ }
+
+ }
+
+ private void findEncodingDetectionParsers(Parser p, List<Parser> encodingDetectionParsers) {
+
+ if (p instanceof CompositeParser) {
+ for (Parser child : ((CompositeParser) p).getAllComponentParsers()) {
+ findEncodingDetectionParsers(child, encodingDetectionParsers);
+ }
+ } else if (p instanceof ParserDecorator) {
+ findEncodingDetectionParsers(((ParserDecorator) p), encodingDetectionParsers);
+ }
+ if (p instanceof AbstractEncodingDetectorParser) {
+ encodingDetectionParsers.add(p);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
index d09d7fc..7759ebd 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
@@ -20,21 +20,13 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.StringWriter;
import org.apache.tika.TikaTest;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.detect.CompositeEncodingDetector;
-import org.apache.tika.detect.EncodingDetector;
-import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.parser.AbstractEncodingDetectorParser;
-import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
@@ -307,25 +299,4 @@ public class TXTParserTest extends TikaTest {
assertEquals("text/x-vcalendar; charset=ISO-8859-1", r.metadata.get(Metadata.CONTENT_TYPE));
}
- @Test
- public void testEncodingDetectorsAreLoaded() {
- EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser)parser).getEncodingDetector();
-
- assertTrue(encodingDetector instanceof CompositeEncodingDetector);
- }
-
- @Test
- public void testEncodingDetectorConfigurability() throws Exception {
- TikaConfig tikaConfig = new TikaConfig(
- getResourceAsStream("/org/apache/tika/config/TIKA-2273-no-icu4j-encoding-detector.xml"));
- AutoDetectParser p = new AutoDetectParser(tikaConfig);
-
- try {
- Metadata metadata = getXML("english.cp500.txt", p).metadata;
- fail("can't detect w/out ICU");
- } catch (TikaException e) {
- assertContains("Failed to detect", e.getMessage());
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-encoding-detector-outside-static-init.xml
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-encoding-detector-outside-static-init.xml b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-encoding-detector-outside-static-init.xml
new file mode 100644
index 0000000..6f70448
--- /dev/null
+++ b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-encoding-detector-outside-static-init.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <!-- exclude TXTParser from Default, add it as if custom
+ and confirm that correct charset detector was added -->
+ <parsers>
+ <parser class="org.apache.tika.parser.DefaultParser">
+ <parser-exclude class="org.apache.tika.parser.txt.TXTParser"/>
+ </parser>
+ <parser class="org.apache.tika.parser.txt.TXTParser">
+ </parser>
+ </parsers>
+ <encodingDetectors>
+ <!-- All detectors except Icu4jEncodingDetector-->
+ <encodingDetector class="org.apache.tika.detect.DefaultEncodingDetector">
+ <encodingDetector-exclude class="org.apache.tika.parser.txt.Icu4jEncodingDetector"/>
+ </encodingDetector>
+ </encodingDetectors>
+</properties>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params-bad-charset.xml
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params-bad-charset.xml b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params-bad-charset.xml
new file mode 100644
index 0000000..42ae7a3
--- /dev/null
+++ b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params-bad-charset.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <!-- Explicitly request default parsers -->
+ <parsers/>
+ <encodingDetectors>
+ <!-- One other detector, to check ordering -->
+ <encodingDetector class="org.apache.tika.detect.NonDetectingEncodingDetector">
+ <params>
+ <param name="charset" type="string">wtf8</param>
+ </params>
+ </encodingDetector>
+ </encodingDetectors>
+</properties>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tika/blob/e7a0c3ee/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params.xml
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params.xml b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params.xml
new file mode 100644
index 0000000..943baf1
--- /dev/null
+++ b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2273-non-detecting-params.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <!-- Explicitly request default parsers -->
+ <parsers/>
+ <encodingDetectors>
+ <!-- One other detector, to check ordering -->
+ <encodingDetector class="org.apache.tika.detect.NonDetectingEncodingDetector">
+ <params>
+ <param name="charset" type="string">UTF-16LE</param>
+ </params>
+ </encodingDetector>
+ </encodingDetectors>
+</properties>
\ No newline at end of file