You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/06/21 18:49:35 UTC
[tika] 02/02: TIKA-2673 -- unit tests for stricter adherence to
spec via Gerard Bouchar
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b688afa01939a1f32775a7e7f0797a8ea466c612
Author: tballison <ta...@mitre.org>
AuthorDate: Thu Jun 21 14:49:20 2018 -0400
TIKA-2673 -- unit tests for stricter adherence to spec via Gerard Bouchar
---
.../tika/parser/html/HtmlEncodingDetector.java | 8 +-
.../tika/parser/html/HtmlEncodingDetectorTest.java | 142 +++++++++++++++++++++
2 files changed, 148 insertions(+), 2 deletions(-)
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
index 84141b9..c4c5188 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
@@ -83,7 +83,7 @@ public class HtmlEncodingDetector implements EncodingDetector {
private static final Pattern HTTP_META_PATTERN = Pattern.compile(
- "(?is)<\\s*meta\\s+([^<>]+)"
+ "(?is)<\\s*meta(?:/|\\s+)([^<>]+)"
);
//this should match both the older:
@@ -97,7 +97,7 @@ public class HtmlEncodingDetector implements EncodingDetector {
//For a more general "not" matcher, try:
//("(?is)charset\\s*=\\s*['\\\"]?\\s*([^<>\\s'\\\";]+)")
private static final Pattern FLEXIBLE_CHARSET_ATTR_PATTERN = Pattern.compile(
- ("(?is)charset\\s*=\\s*(?:['\\\"]\\s*)?([-_:\\.a-z0-9]+)")
+ ("(?is)\\bcharset\\s*=\\s*(?:['\\\"]\\s*)?([-_:\\.a-z0-9]+)")
);
private static final Charset ASCII = Charset.forName("US-ASCII");
@@ -154,6 +154,10 @@ public class HtmlEncodingDetector implements EncodingDetector {
if (CHARSETS_UNSUPPORTED_BY_IANA.contains(candCharset.toLowerCase(Locale.US))) {
continue;
}
+ if ("x-user-defined".equalsIgnoreCase(candCharset)) {
+ candCharset = "windows-1252";
+ }
+
if (CharsetUtils.isSupported(candCharset)) {
try {
return CharsetUtils.forName(candCharset);
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
new file mode 100644
index 0000000..40ab01c
--- /dev/null
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.parser.html;
+
+
+import org.apache.tika.metadata.Metadata;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.*;
+
+public class HtmlEncodingDetectorTest {
+
+ @Test
+ public void basic() throws IOException {
+ assertWindows1252("<meta charset='WINDOWS-1252'>");
+ }
+
+ @Test
+ @Ignore("can we can prove this harms detection")
+ public void utf16() throws IOException {
+ // According to the specification 'If charset is a UTF-16 encoding, then set charset to UTF-8.'
+ assertCharset("<meta charset='UTF-16BE'>", StandardCharsets.UTF_8);
+ }
+
+ @Test
+ public void xUserDefined() throws IOException {
+ // According to the specification 'If charset is x-user-defined, then set charset to windows-1252.'
+ assertWindows1252("<meta charset='x-user-defined'>");
+ }
+
+ @Test
+ public void withSlash() throws IOException {
+ assertWindows1252("<meta/charset='WINDOWS-1252'>");
+ }
+
+ @Test
+ @Ignore("until we do a full parse")
+ public void insideTag() throws IOException {
+ assertWindows1252("<meta name='description'" +
+ "content='If I write charset=UTF-8 here, it doesnt mean the page is in UTF-8'/>" +
+ "<meta charset='WINDOWS-1252'>");
+ }
+
+ @Test
+ @Ignore("until we do a full parse")
+ public void missingAttribute() throws IOException {
+ assertWindows1252(
+ "<meta content='charset=UTF-8'>" + // missing http-equiv attribute
+ "<meta charset='WINDOWS-1252'>" // valid declaration
+ );
+ }
+
+ @Test
+ @Ignore("until we do a full parse")
+ public void insideSpecialTag() throws IOException {
+ // Content inside <?, <!, and </ should be ignored
+ for (byte b : "?!/".getBytes())
+ assertWindows1252(
+ "<" + (char) b + // start comment
+ "<meta charset='UTF-8'>" + // inside special tag
+ "<meta charset='WINDOWS-1252'>" // real charset declaration
+ );
+ }
+
+ @Test
+ @Ignore("until we can prove this harms detection")
+ public void spaceBeforeTag() throws IOException {
+ assertWindows1252(
+ "< meta charset='UTF-8'>" + // invalid charset declaration
+ "<meta charset='WINDOWS-1252'>" // real charset declaration
+ );
+ }
+
+ @Test
+ public void invalidAttribute() throws IOException {
+ assertWindows1252(
+ "<meta " +
+ "badcharset='UTF-8' " + // invalid charset declaration
+ "charset='WINDOWS-1252'>" // real charset declaration
+ );
+ }
+
+ @Test
+ @Ignore("until we can prove this harms detection")
+ public void unmatchedQuote() throws IOException {
+ assertWindows1252(
+ "<meta http-equiv='content-type' content='charset=\"UTF-8'>" + // invalid charset declaration
+ "<meta charset='WINDOWS-1252'>" // real charset declaration
+ );
+ }
+
+
+ @Test
+ @Ignore("until we do a full parse")
+ public void withCompactComment() throws IOException {
+ // <!--> is a valid comment
+ assertWindows1252(
+ "<!--" + // start comment
+ "<meta charset='UTF-8'>" + // inside comment
+ "-->" + // end comment
+ "<!-->" + // compact comment
+ "<meta charset='WINDOWS-1252'>" // outside comment, charset declaration
+ );
+ }
+
+ private void assertWindows1252(String html) throws IOException {
+ assertCharset(html, Charset.forName("WINDOWS-1252"));
+ }
+
+ private void assertCharset(String html, Charset charset) throws IOException {
+ assertEquals(html + " should be detected as " + charset,
+ charset, detectCharset(html));
+ }
+
+ private Charset detectCharset(String test) throws IOException {
+ Metadata metadata = new Metadata();
+ InputStream inStream = new ByteArrayInputStream(test.getBytes(StandardCharsets.UTF_8));
+ return new HtmlEncodingDetector().detect(inStream, metadata);
+ }
+}
\ No newline at end of file