You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2017/11/02 13:41:18 UTC

[tika] branch master updated: TIKA-2485 -- Allow configuration of markLimit in EncodingDetectors via tika-config.xml

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/master by this push:
     new c009dc7   TIKA-2485 -- Allow configuration of markLimit in EncodingDetectors via tika-config.xml
c009dc7 is described below

commit c009dc71cc8428e0a752100af7e9d18d7e5e3096
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Nov 2 09:41:10 2017 -0400

     TIKA-2485 -- Allow configuration of markLimit in EncodingDetectors via tika-config.xml
---
 CHANGES.txt                                        |  3 +
 .../tika/parser/html/HtmlEncodingDetector.java     | 24 +++++++-
 .../apache/tika/parser/txt/CharsetDetector.java    | 17 ++++-
 .../tika/parser/txt/Icu4jEncodingDetector.java     | 20 +++++-
 .../tika/parser/txt/UniversalEncodingDetector.java | 25 ++++++--
 .../tika/config/TikaEncodingDetectorTest.java      | 72 ++++++++++++++++++++++
 .../TIKA-2485-encoding-detector-mark-limits.xml    | 38 ++++++++++++
 7 files changed, 188 insertions(+), 11 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 74e486f..d06488b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,8 @@
 Release 1.17 - ???
 
+  * Allow configuration of markLimit in EncodingDetectors
+    via tika-config.xml (TIKA-2485).
+
   * RFC822Parser now selects the best alternative for
     multipart/alternative body components.  This aligns with the
     behavior of the OutlookParser (TIKA-2478).  Users can select
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
index 1ebe076..559ec4d 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
@@ -23,6 +23,7 @@ import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.tika.config.Field;
 import org.apache.tika.detect.EncodingDetector;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.utils.CharsetUtils;
@@ -39,7 +40,7 @@ import org.apache.tika.utils.CharsetUtils;
 public class HtmlEncodingDetector implements EncodingDetector {
 
     // TIKA-357 - use bigger buffer for meta tag sniffing (was 4K)
-    private static final int META_TAG_BUFFER_SIZE = 8192;
+    private static final int DEFAULT_MARK_LIMIT = 8192;
 
 
     private static final Pattern HTTP_META_PATTERN = Pattern.compile(
@@ -62,6 +63,9 @@ public class HtmlEncodingDetector implements EncodingDetector {
 
     private static final Charset ASCII = Charset.forName("US-ASCII");
 
+    @Field
+    private int markLimit = DEFAULT_MARK_LIMIT;
+
     public Charset detect(InputStream input, Metadata metadata)
             throws IOException {
         if (input == null) {
@@ -69,8 +73,8 @@ public class HtmlEncodingDetector implements EncodingDetector {
         }
 
         // Read enough of the text stream to capture possible meta tags
-        input.mark(META_TAG_BUFFER_SIZE);
-        byte[] buffer = new byte[META_TAG_BUFFER_SIZE];
+        input.mark(markLimit);
+        byte[] buffer = new byte[markLimit];
         int n = 0;
         int m = input.read(buffer);
         while (m != -1 && n < buffer.length) {
@@ -120,4 +124,18 @@ public class HtmlEncodingDetector implements EncodingDetector {
         return null;
     }
 
+    /**
+     * How far into the stream to read for charset detection.
+     * Default is 8192.
+     *
+     * @param markLimit
+     */
+    @Field
+    public void setMarkLimit(int markLimit) {
+        this.markLimit = markLimit;
+    }
+
+    public int getMarkLimit() {
+        return markLimit;
+    }
 }
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
index 172cb46..b1906e9 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/CharsetDetector.java
@@ -53,8 +53,10 @@ public class CharsetDetector {
 //   actually choose the "real" charset.  All assuming that the application just
 //   wants the data, and doesn't care about a char set name.
 
-    private static final int kBufSize = 12000;//This is a Tika modification; ICU's is 8000
     private static final int MAX_CONFIDENCE = 100;
+
+    static final int DEFAULT_MARK_LIMIT = 12000; //This is a Tika modification; ICU's is 8000
+
     /*
      * List of recognizers for all charsets known to the implementation.
      */
@@ -112,8 +114,9 @@ public class CharsetDetector {
      *     the recognition process
      *
      */
-    byte[] fInputBytes =       // The text to be checked.  Markup will have been
-            new byte[kBufSize];  //   removed if appropriate.
+    final byte[] fInputBytes;  // The text to be checked.  Markup will have been
+                               // removed if appropriate.
+
     int fInputLen;          // Length of the byte data in fInputBytes.
     short fByteStats[] =      // byte frequency statistics for the input text.
             new short[256];  //   Value is percent, not absolute.
@@ -133,12 +136,20 @@ public class CharsetDetector {
             false;
     private boolean[] fEnabledRecognizers;   // If not null, active set of charset recognizers had
 
+    private final int kBufSize;
+
     /**
      * Constructor
      *
      * @stable ICU 3.4
      */
     public CharsetDetector() {
+        this(DEFAULT_MARK_LIMIT);
+    }
+
+    public CharsetDetector(int markLimit) {
+        kBufSize = markLimit;
+        fInputBytes = new byte[kBufSize];
     }
 
     /**
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
index b811a1d..4a2c56b 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/Icu4jEncodingDetector.java
@@ -31,13 +31,16 @@ public class Icu4jEncodingDetector implements EncodingDetector {
     @Field
     private boolean stripMarkup = false;
 
+    @Field
+    private int markLimit = CharsetDetector.DEFAULT_MARK_LIMIT;
+
     public Charset detect(InputStream input, Metadata metadata)
             throws IOException {
         if (input == null) {
             return null;
         }
 
-        CharsetDetector detector = new CharsetDetector();
+        CharsetDetector detector = new CharsetDetector(markLimit);
 
         String incomingCharset = metadata.get(Metadata.CONTENT_ENCODING);
         String incomingType = metadata.get(Metadata.CONTENT_TYPE);
@@ -94,4 +97,19 @@ public class Icu4jEncodingDetector implements EncodingDetector {
     public boolean getStripMarkup() {
         return stripMarkup;
     }
+
+    /**
+     * How far into the stream to read for charset detection.
+     * Default is 12000.
+     *
+     * @param markLimit
+     */
+    @Field
+    public void setMarkLimit(int markLimit) {
+        this.markLimit = markLimit;
+    }
+
+    public int getMarkLimit() {
+        return markLimit;
+    }
 }
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
index fc3fb04..1b9108a 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/txt/UniversalEncodingDetector.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
 
+import org.apache.tika.config.Field;
 import org.apache.tika.detect.EncodingDetector;
 import org.apache.tika.metadata.Metadata;
 
@@ -27,7 +28,9 @@ public class UniversalEncodingDetector implements EncodingDetector {
 
     private static final int BUFSIZE = 1024;
 
-    private static final int LOOKAHEAD = 16 * BUFSIZE;
+    private static final int DEFAULT_MARK_LIMIT = 16 * BUFSIZE;
+
+    private int markLimit = DEFAULT_MARK_LIMIT;
 
     public Charset detect(InputStream input, Metadata metadata)
             throws IOException {
@@ -35,7 +38,7 @@ public class UniversalEncodingDetector implements EncodingDetector {
             return null;
         }
 
-        input.mark(LOOKAHEAD);
+        input.mark(markLimit);
         try {
             UniversalEncodingListener listener =
                     new UniversalEncodingListener(metadata);
@@ -43,10 +46,10 @@ public class UniversalEncodingDetector implements EncodingDetector {
             byte[] b = new byte[BUFSIZE];
             int n = 0;
             int m = input.read(b);
-            while (m != -1 && n < LOOKAHEAD && !listener.isDone()) {
+            while (m != -1 && n < markLimit && !listener.isDone()) {
                 n += m;
                 listener.handleData(b, 0, m);
-                m = input.read(b, 0, Math.min(b.length, LOOKAHEAD - n));
+                m = input.read(b, 0, Math.min(b.length,markLimit - n));
             }
 
             return listener.dataEnd();
@@ -57,4 +60,18 @@ public class UniversalEncodingDetector implements EncodingDetector {
         }
     }
 
+    /**
+     * How far into the stream to read for charset detection.
+     * Default is 8192.
+     *
+     * @param markLimit
+     */
+    @Field
+    public void setMarkLimit(int markLimit) {
+        this.markLimit = markLimit;
+    }
+
+    public int getMarkLimit() {
+        return markLimit;
+    }
 }
diff --git a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
index f23eab4..31d82d5 100644
--- a/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/config/TikaEncodingDetectorTest.java
@@ -25,12 +25,14 @@ import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
+import jj2000.j2k.codestream.HeaderInfo;
 import org.apache.tika.Tika;
 import org.apache.tika.detect.CompositeEncodingDetector;
 import org.apache.tika.detect.EncodingDetector;
 import org.apache.tika.detect.NonDetectingEncodingDetector;
 import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.parser.AbstractEncodingDetectorParser;
 import org.apache.tika.parser.AutoDetectParser;
@@ -178,6 +180,76 @@ public class TikaEncodingDetectorTest extends AbstractTikaConfigTest {
 
     }
 
+    @Test
+    public void testMarkLimitUnit() throws Exception {
+        TikaConfig tikaConfig = new TikaConfig(
+                getResourceAsStream("/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml"));
+        AutoDetectParser p = new AutoDetectParser(tikaConfig);
+        List<Parser> parsers = new ArrayList<>();
+        findEncodingDetectionParsers(p, parsers);
+
+        assertEquals(3, parsers.size());
+        for (Parser childParser : parsers) {
+            EncodingDetector encodingDetector = ((AbstractEncodingDetectorParser)childParser).getEncodingDetector();
+            assertTrue(encodingDetector instanceof CompositeEncodingDetector);
+            assertEquals(3, ((CompositeEncodingDetector) encodingDetector).getDetectors().size());
+            for (EncodingDetector childEncodingDetector : ((CompositeEncodingDetector)encodingDetector).getDetectors()) {
+                if (childEncodingDetector instanceof HtmlEncodingDetector) {
+                    assertEquals(childParser.getClass().toString(), 64000, ((HtmlEncodingDetector) childEncodingDetector).getMarkLimit());
+                } else if (childEncodingDetector instanceof  UniversalEncodingDetector) {
+                    assertEquals(childParser.getClass().toString(), 64001, ((UniversalEncodingDetector)childEncodingDetector).getMarkLimit());
+                } else if (childEncodingDetector instanceof Icu4jEncodingDetector) {
+                    assertEquals(childParser.getClass().toString(), 64002, ((Icu4jEncodingDetector)childEncodingDetector).getMarkLimit());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testMarkLimitIntegration() throws Exception {
+        StringBuilder sb = new StringBuilder();
+        sb.append("<html><head><script>");
+        for (int i = 0; i < 4000; i++) { //script length = 20000
+            sb.append("blah ");
+        }
+        sb.append("</script>");
+        sb.append("<meta charset=\"utf-8\">").append("</head><body>");
+        sb.append("<p>Frugt, gr&#248;nd og mere</p>");
+        sb.append("<p>5 kg \u00f8kologisk frugt og gr\u00F8nt, lige til at g\u00E5 til</p>");
+        sb.append("</body></html>");
+
+        byte[] bytes = sb.toString().getBytes(StandardCharsets.UTF_8);
+
+        //assert default fails -- will need to fix this if we change our defaults!
+        AutoDetectParser p = new AutoDetectParser();
+
+        Metadata metadata = new Metadata();
+        String xml = getXML(
+                TikaInputStream.get(bytes),
+                p, metadata).xml;
+
+        assertContains("gr\u00F8nd", xml);
+        assertNotContained("\u00f8kologisk", xml);
+        assertNotContained("gr\u00F8nt", xml);
+        assertNotContained("g\u00E5 til", xml);
+
+        //now test that fix works
+        TikaConfig tikaConfig = new TikaConfig(
+                getResourceAsStream("/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml"));
+        p = new AutoDetectParser(tikaConfig);
+
+        metadata = new Metadata();
+        xml = getXML(
+                TikaInputStream.get(bytes),
+                p, metadata).xml;
+
+        assertContains("gr\u00F8nd", xml);
+        assertContains("\u00f8kologisk", xml);
+        assertContains("gr\u00F8nt", xml);
+        assertContains("g\u00E5 til", xml);
+    }
+
+
     private void findEncodingDetectionParsers(Parser p, List<Parser> encodingDetectionParsers) {
 
         if (p instanceof CompositeParser) {
diff --git a/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml
new file mode 100644
index 0000000..7ff326d
--- /dev/null
+++ b/tika-parsers/src/test/resources/org/apache/tika/config/TIKA-2485-encoding-detector-mark-limits.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<properties>
+    <!-- Explicitly request default parsers -->
+    <parsers/>
+    <encodingDetectors>
+        <encodingDetector class="org.apache.tika.parser.html.HtmlEncodingDetector">
+            <params>
+                <param name="markLimit" type="int">64000</param>
+            </params>
+        </encodingDetector>
+        <encodingDetector class="org.apache.tika.parser.txt.UniversalEncodingDetector">
+            <params>
+                <param name="markLimit" type="int">64001</param>
+            </params>
+        </encodingDetector>
+        <encodingDetector class="org.apache.tika.parser.txt.Icu4jEncodingDetector">
+            <params>
+                <param name="markLimit" type="int">64002</param>
+            </params>
+        </encodingDetector>
+    </encodingDetectors>
+</properties>
\ No newline at end of file

-- 
To stop receiving notification emails like this one, please contact
['"commits@tika.apache.org" <co...@tika.apache.org>'].