You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/11/02 13:41:18 UTC
[tika] branch master updated: TIKA-2485 -- Allow configuration of
markLimit in EncodingDetectors via tika-config.xml
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/master by this push:
new c009dc7 TIKA-2485 -- Allow configuration of markLimit in EncodingDetectors via tika-config.xml
c009dc7 is described below
commit c009dc71cc8428e0a752100af7e9d18d7e5e3096
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Nov 2 09:41:10 2017 -0400
TIKA-2485 -- Allow configuration of markLimit in EncodingDetectors via tika-config.xml
---
CHANGES.txt | 3 +
.../tika/parser/html/HtmlEncodingDetector.java | 24 +++++++-
.../apache/tika/parser/txt/CharsetDetector.java | 17 ++++-
.../tika/parser/txt/Icu4jEncodingDetector.java | 20 +++++-
.../tika/parser/txt/UniversalEncodingDetector.java | 25 ++++++--
.../tika/config/TikaEncodingDetectorTest.java | 72 ++++++++++++++++++++++
.../TIKA-2485-encoding-detector-mark-limits.xml | 38 ++++++++++++
7 files changed, 188 insertions(+), 11 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 74e486f..d06488b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,8 @@
Release 1.17 - ???
+ * Allow configuration of markLimit in EncodingDetectors
+ via tika-config.xml (TIKA-2485).
+
* RFC822Parser now selects the best alternative for
multipart/alternative body components. This aligns with the
behavior of the OutlookParser (TIKA-2478). Users can select
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
index 1ebe076..559ec4d 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
@@ -23,6 +23,7 @@ import java.nio.charset.Charset;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.tika.config.Field;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.utils.CharsetUtils;
@@ -39,7 +40,7 @@ import org.apache.tika.utils.CharsetUtils;
public class HtmlEncodingDetector implements EncodingDetector {
// TIKA-357 - use bigger buffer for meta tag sniffing (was 4K)
- private static final int META_TAG_BUFFER_SIZE = 8192;
+ private static final int DEFAULT_MARK_LIMIT = 8192;
private static final Pattern HTTP_META_PATTERN = Pattern.compile(
@@ -62,6 +63,9 @@ public class HtmlEncodingDetector implements EncodingDetector {
private static final Charset ASCII = Charset.forName("US-ASCII");
+ @Field
+ private int markLimit = DEFAULT_MARK_LIMIT;
+
public Charset detect(InputStream input, Metadata metadata)
throws IOException {
if (input == null) {
@@ -69,8 +73,8 @@ public class HtmlEncodingDetector implements EncodingDetector {
}
// Read enough of the text stream to capture possible meta tags
- input.mark(META_TAG_BUFFER_SIZE);
- byte[] buffer = new byte[META_TAG_BUFFER_SIZE];
+ input.mark(markLimit);
+ byte[] buffer = new byte[markLimit];
int n = 0;
int m = input.read(buffer);
while (m != -1 && n < buffer.length) {
@@ -120,4 +124,18 @@ public class HtmlEncodingDetector implements EncodingDetector {
return null;
}
+ /**
+ * How far into the stream to read for charset detection.
+ * Default is 8192.
+ *
+ * @param markLimit
+ */
+ @Field
+ public void setMarkLimit(int markLimit) {
+ this.markLimit = markLimit;
+ }
+
+ public int getMarkLimit() {
+ return markLimit;
+ }
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
index 172cb46..b1906e9 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
@@ -53,8 +53,10 @@ public class CharsetDetector {
// actually choose the "real" charset. All assuming that the application just
// wants the data, and doesn't care about a char set name.
- private static final int kBufSize = 12000;//This is a Tika modification; ICU's is 8000
private static final int MAX_CONFIDENCE = 100;
+
+ static final int DEFAULT_MARK_LIMIT = 12000; //This is a Tika modification; ICU's is 8000
+
/*
* List of recognizers for all charsets known to the implementation.
*/
@@ -112,8 +114,9 @@ public class CharsetDetector {
* the recognition process
*
*/
- byte[] fInputBytes = // The text to be checked. Markup will have been
- new byte[kBufSize]; // removed if appropriate.
+ final byte[] fInputBytes; // The text to be checked. Markup will have been
+ // removed if appropriate.
+
int fInputLen; // Length of the byte data in fInputBytes.
short fByteStats[] = // byte frequency statistics for the input text.
new short[256]; // Value is percent, not absolute.
@@ -133,12 +136,20 @@ public class CharsetDetector {
false;
private boolean[] fEnabledRecognizers; // If not null, active set of charset recognizers had
+ private final int kBufSize;
+
/**
* Constructor
*
* @stable ICU 3.4
*/
public CharsetDetector() {
+ this(DEFAULT_MARK_LIMIT);
+ }
+
+ public CharsetDetector(int markLimit) {
+ kBufSize = markLimit;
+ fInputBytes = new byte[kBufSize];
}
/**
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
index b811a1d..4a2c56b 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
@@ -31,13 +31,16 @@ public class Icu4jEncodingDetector implements EncodingDetector {
@Field
private boolean stripMarkup = false;
+ @Field
+ private int markLimit = CharsetDetector.DEFAULT_MARK_LIMIT;
+
public Charset detect(InputStream input, Metadata metadata)
throws IOException {
if (input == null) {
return null;
}
- CharsetDetector detector = new CharsetDetector();
+ CharsetDetector detector = new CharsetDetector(markLimit);
String incomingCharset = metadata.get(Metadata.CONTENT_ENCODING);
String incomingType = metadata.get(Metadata.CONTENT_TYPE);
@@ -94,4 +97,19 @@ public class Icu4jEncodingDetector implements EncodingDetector {
public boolean getStripMarkup() {
return stripMarkup;
}
+
+ /**
+ * How far into the stream to read for charset detection.
+ * Default is 12000.
+ *
+ * @param markLimit
+ */
+ @Field
+ public void setMarkLimit(int markLimit) {
+ this.markLimit = markLimit;
+ }
+
+ public int getMarkLimit() {
+ return markLimit;
+ }
}
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
index fc3fb04..1b9108a 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
+import org.apache.tika.config.Field;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.metadata.Metadata;
@@ -27,7 +28,9 @@ public class UniversalEncodingDetector implements EncodingDetector {
private static final int BUFSIZE = 1024;
- private static final int LOOKAHEAD = 16 * BUFSIZE;
+ private static final int DEFAULT_MARK_LIMIT = 16 * BUFSIZE;
+
+ private int markLimit = DEFAULT_MARK_LIMIT;
public Charset detect(InputStream input, Metadata metadata)
throws IOException {
@@ -35,7 +38,7 @@ public class UniversalEncodingDetector implements EncodingDetector {
return null;
}
- input.mark(LOOKAHEAD);
+ input.mark(markLimit);
try {
UniversalEncodingListener listener =
new UniversalEncodingListener(metadata);
@@ -43,10 +46,10 @@ public class UniversalEncodingDetector implements EncodingDetector {
byte[] b = new byte[BUFSIZE];
int n = 0;
int m = input.read(b);
- while (m != -1 && n < LOOKAHEAD && !listener.isDone()) {
+ while (m != -1 && n < markLimit && !listener.isDone()) {
n += m;
listener.handleData(b, 0, m);
- m = input.read(b, 0, Math.min(b.length, LOOKAHEAD - n));
+ m = input.read(b, 0, Math.min(b.length,markLimit - n));
}
return listener.dataEnd();
@@ -57,4 +60,18 @@ public class UniversalEncodingDetector implements EncodingDetector {
}
}
+ /**
+ * How far into the stream to read for charset detection.
+ * Default is 8192.
+ *
+ * @param markLimit
+ */
+ @Field
+ public void setMarkLimit(int markLimit) {
+ this.markLimit = markLimit;
+ }
+
+ public int getMarkLimit() {
+ return markLimit;
+ }
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index f23eab4..31d82d5 100644
--- a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -25,12 +25,14 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import jj2000.j2k.codestream.HeaderInfo;
import org.apache.tika.Tika;
import org.apache.tika.detect.CompositeEncodingDetector;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.detect.NonDetectingEncodingDetector;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AbstractEncodingDetectorParser;
import org.apache.tika.parser.AutoDetectParser;
@@ -178,6 +180,76 @@ public class TikaEncodingDetectorTest extends AbstractTikaConfigTest {
}
+ @Test
+ public void testMarkLimitUnit() throws Exception {
+ TikaConfig tikaConfig = new TikaConfig(
+ getResourceAsStream("/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml"));
+ AutoDetectParser p = new AutoDetectParser(tikaConfig);
+ List<Parser> parsers = new ArrayList<>();
+ findEncodingDetectionParsers(p, parsers);
+
+ assertEquals(3, parsers.size());
+ for (Parser childParser : parsers) {
+ EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser)childParser).getEncodingDetector();
+ assertTrue(encodingDetector instanceof CompositeEncodingDetector);
+ assertEquals(3, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
+ for (EncodingDetector childEncodingDetector : ((CompositeEncodingDetector)encodingDetector).getDetectors()) {
+ if (childEncodingDetector instanceof HtmlEncodingDetector) {
+ assertEquals(childParser.getClass().toString(), 64000, ((HtmlEncodingDetector) childEncodingDetector).getMarkLimit());
+ } else if (childEncodingDetector instanceof UniversalEncodingDetector) {
+ assertEquals(childParser.getClass().toString(), 64001, ((UniversalEncodingDetector)childEncodingDetector).getMarkLimit());
+ } else if (childEncodingDetector instanceof Icu4jEncodingDetector) {
+ assertEquals(childParser.getClass().toString(), 64002, ((Icu4jEncodingDetector)childEncodingDetector).getMarkLimit());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMarkLimitIntegration() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<html><head><script>");
+ for (int i = 0; i < 4000; i++) { //script length = 20000
+ sb.append("blah ");
+ }
+ sb.append("</script>");
+ sb.append("<meta charset=\"utf-8\">").append("</head><body>");
+ sb.append("<p>Frugt, grønd og mere</p>");
+ sb.append("<p>5 kg \u00f8kologisk frugt og gr\u00F8nt, lige til at g\u00E5 til</p>");
+ sb.append("</body></html>");
+
+ byte[] bytes = sb.toString().getBytes(StandardCharsets.UTF_8);
+
+ //assert default fails -- will need to fix this if we change our defaults!
+ AutoDetectParser p = new AutoDetectParser();
+
+ Metadata metadata = new Metadata();
+ String xml = getXML(
+ TikaInputStream.get(bytes),
+ p, metadata).xml;
+
+ assertContains("gr\u00F8nd", xml);
+ assertNotContained("\u00f8kologisk", xml);
+ assertNotContained("gr\u00F8nt", xml);
+ assertNotContained("g\u00E5 til", xml);
+
+ //now test that fix works
+ TikaConfig tikaConfig = new TikaConfig(
+ getResourceAsStream("/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml"));
+ p = new AutoDetectParser(tikaConfig);
+
+ metadata = new Metadata();
+ xml = getXML(
+ TikaInputStream.get(bytes),
+ p, metadata).xml;
+
+ assertContains("gr\u00F8nd", xml);
+ assertContains("\u00f8kologisk", xml);
+ assertContains("gr\u00F8nt", xml);
+ assertContains("g\u00E5 til", xml);
+ }
+
+
private void findEncodingDetectionParsers(Parser p, List<Parser> encodingDetectionParsers) {
if (p instanceof CompositeParser) {
diff --git a/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml
new file mode 100644
index 0000000..7ff326d
--- /dev/null
+++ b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<properties>
+ <!-- Explicitly request default parsers -->
+ <parsers/>
+ <encodingDetectors>
+ <encodingDetector class="org.apache.tika.parser.html.HtmlEncodingDetector">
+ <params>
+ <param name="markLimit" type="int">64000</param>
+ </params>
+ </encodingDetector>
+ <encodingDetector class="org.apache.tika.parser.txt.UniversalEncodingDetector">
+ <params>
+ <param name="markLimit" type="int">64001</param>
+ </params>
+ </encodingDetector>
+ <encodingDetector class="org.apache.tika.parser.txt.Icu4jEncodingDetector">
+ <params>
+ <param name="markLimit" type="int">64002</param>
+ </params>
+ </encodingDetector>
+ </encodingDetectors>
+</properties>
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].