You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2012/07/09 01:27:43 UTC

svn commit: r1358872 - in /tika/trunk/tika-parsers/src: main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java test/java/org/apache/tika/parser/html/HtmlParserTest.java

Author: jukka
Date: Sun Jul  8 23:27:42 2012
New Revision: 1358872

URL: http://svn.apache.org/viewvc?rev=1358872&view=rev
Log:
TIKA-892: Tika does not use the HTML5 meta charset tag when determining charset

Based on the patch by Chris Jones

Modified:
    tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
    tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java

Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java?rev=1358872&r1=1358871&r2=1358872&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java Sun Jul  8 23:27:42 2012
@@ -19,7 +19,6 @@ package org.apache.tika.parser.html;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -48,6 +47,9 @@ public class HtmlEncodingDetector implem
             + "Content-Type['\\\"]\\s+content\\s*=\\s*['\\\"]"
             + "([^'\\\"]+)['\\\"]");
 
+    private static final Pattern META_CHARSET_PATTERN = Pattern.compile(
+            "(?is)<meta\\s+charset\\s*=\\s*['\\\"]([^'\\\"]+)['\\\"]");
+
     private static final Charset ASCII = Charset.forName("US-ASCII");
 
     public Charset detect(InputStream input, Metadata metadata)
@@ -67,20 +69,31 @@ public class HtmlEncodingDetector implem
         }
         input.reset();
 
-        // Interpret the head as ASCII and try to spot a http-equiv setting
-        CharBuffer head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n));
-        Matcher matcher = HTTP_EQUIV_PATTERN.matcher(head.toString());
-        if (matcher.find()) {
-            MediaType type = MediaType.parse(matcher.group(1));
+        // Interpret the head as ASCII and try to spot a meta tag with
+        // a possible character encoding hint
+        String charset = null;
+        String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString();
+
+        Matcher equiv = HTTP_EQUIV_PATTERN.matcher(head);
+        if (equiv.find()) {
+            MediaType type = MediaType.parse(equiv.group(1));
             if (type != null) {
-                String charset = type.getParameters().get("charset");
-                if (charset != null) {
-                    try {
-                        return CharsetUtils.forName(charset);
-                    } catch (Exception e) {
-                        // ignore
-                    }
-                }
+                charset = type.getParameters().get("charset");
+            }
+        }
+        if (charset == null) {
+            // TIKA-892: HTML5 meta charset tag
+            Matcher meta = META_CHARSET_PATTERN.matcher(head);
+            if (meta.find()) {
+                charset = meta.group(1);
+            }
+        }
+
+        if (charset != null) {
+            try {
+                return CharsetUtils.forName(charset);
+            } catch (Exception e) {
+                // ignore
             }
         }
 

Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java?rev=1358872&r1=1358871&r2=1358872&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java Sun Jul  8 23:27:42 2012
@@ -249,6 +249,22 @@ public class HtmlParserTest extends Test
     }
 
     /**
+     * Test case for TIKA-892
+     * @see <a href="https://issues.apache.org/jira/browse/TIKA-892">TIKA-892</a>
+     */
+    public void testHtml5Charset() throws Exception {
+        String test =
+                "<html><head><meta charset=\"ISO-8859-15\" />"
+                + "<title>the name is \u00e1ndre</title>"
+                + "</head><body></body></html>";
+        Metadata metadata = new Metadata();
+        new HtmlParser().parse(
+                new ByteArrayInputStream(test.getBytes("ISO-8859-1")),
+                new BodyContentHandler(), metadata, new ParseContext());
+        assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING));
+    }
+
+    /**
      * Test case for TIKA-334
      * @see <a href="https://issues.apache.org/jira/browse/TIKA-334">TIKA-334</a>
      */