You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ju...@apache.org on 2012/07/09 01:27:43 UTC
svn commit: r1358872 - in /tika/trunk/tika-parsers/src:
main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
test/java/org/apache/tika/parser/html/HtmlParserTest.java
Author: jukka
Date: Sun Jul 8 23:27:42 2012
New Revision: 1358872
URL: http://svn.apache.org/viewvc?rev=1358872&view=rev
Log:
TIKA-892: Tika does not use the HTML5 meta charset tag when determining charset
Based on the patch by Chris Jones
Modified:
tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
Modified: tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java?rev=1358872&r1=1358871&r2=1358872&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java (original)
+++ tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java Sun Jul 8 23:27:42 2012
@@ -19,7 +19,6 @@ package org.apache.tika.parser.html;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -48,6 +47,9 @@ public class HtmlEncodingDetector implem
+ "Content-Type['\\\"]\\s+content\\s*=\\s*['\\\"]"
+ "([^'\\\"]+)['\\\"]");
+ private static final Pattern META_CHARSET_PATTERN = Pattern.compile(
+ "(?is)<meta\\s+charset\\s*=\\s*['\\\"]([^'\\\"]+)['\\\"]");
+
private static final Charset ASCII = Charset.forName("US-ASCII");
public Charset detect(InputStream input, Metadata metadata)
@@ -67,20 +69,31 @@ public class HtmlEncodingDetector implem
}
input.reset();
- // Interpret the head as ASCII and try to spot a http-equiv setting
- CharBuffer head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n));
- Matcher matcher = HTTP_EQUIV_PATTERN.matcher(head.toString());
- if (matcher.find()) {
- MediaType type = MediaType.parse(matcher.group(1));
+ // Interpret the head as ASCII and try to spot a meta tag with
+ // a possible character encoding hint
+ String charset = null;
+ String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString();
+
+ Matcher equiv = HTTP_EQUIV_PATTERN.matcher(head);
+ if (equiv.find()) {
+ MediaType type = MediaType.parse(equiv.group(1));
if (type != null) {
- String charset = type.getParameters().get("charset");
- if (charset != null) {
- try {
- return CharsetUtils.forName(charset);
- } catch (Exception e) {
- // ignore
- }
- }
+ charset = type.getParameters().get("charset");
+ }
+ }
+ if (charset == null) {
+ // TIKA-892: HTML5 meta charset tag
+ Matcher meta = META_CHARSET_PATTERN.matcher(head);
+ if (meta.find()) {
+ charset = meta.group(1);
+ }
+ }
+
+ if (charset != null) {
+ try {
+ return CharsetUtils.forName(charset);
+ } catch (Exception e) {
+ // ignore
}
}
Modified: tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java?rev=1358872&r1=1358871&r2=1358872&view=diff
==============================================================================
--- tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java (original)
+++ tika/trunk/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java Sun Jul 8 23:27:42 2012
@@ -249,6 +249,22 @@ public class HtmlParserTest extends Test
}
/**
+ * Test case for TIKA-892
+ * @see <a href="https://issues.apache.org/jira/browse/TIKA-892">TIKA-892</a>
+ */
+ public void testHtml5Charset() throws Exception {
+ String test =
+ "<html><head><meta charset=\"ISO-8859-15\" />"
+ + "<title>the name is \u00e1ndre</title>"
+ + "</head><body></body></html>";
+ Metadata metadata = new Metadata();
+ new HtmlParser().parse(
+ new ByteArrayInputStream(test.getBytes("ISO-8859-1")),
+ new BodyContentHandler(), metadata, new ParseContext());
+ assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING));
+ }
+
+ /**
* Test case for TIKA-334
* @see <a href="https://issues.apache.org/jira/browse/TIKA-334">TIKA-334</a>
*/