You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2016/05/26 14:50:12 UTC
[1/2] tika git commit: TIKA-1985: fix auto-import mis-clean up
Repository: tika
Updated Branches:
refs/heads/master dcaeccbab -> 16290d86c
TIKA-1985: fix auto-import mis-clean up
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/1a04f805
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/1a04f805
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/1a04f805
Branch: refs/heads/master
Commit: 1a04f8056f0140a34641435f4aff21443efa7bbd
Parents: dcaeccb
Author: tballison <ta...@mitre.org>
Authored: Thu May 26 10:27:33 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Thu May 26 10:27:33 2016 -0400
----------------------------------------------------------------------
.../main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/1a04f805/tika-parsers/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java b/tika-parsers/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
index 7c3e52e..14d2bbf 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/dbf/DBFColumnHeader.java
@@ -17,13 +17,13 @@
package org.apache.tika.parser.dbf;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.tika.parser.dbf.DBFColumnHeader.ColType.AT;
import static org.apache.tika.parser.dbf.DBFColumnHeader.ColType.NULL;
import static org.apache.tika.parser.dbf.DBFColumnHeader.ColType.PLUS;
-import static org.json.zip.JSONzip.end;
class DBFColumnHeader {
@@ -71,7 +71,8 @@ class DBFColumnHeader {
public void setType(int type) {
colType = COL_TYPE_MAP.get(type);
if (colType == null) {
- throw new IllegalArgumentException("Unrecognized column type for column: " + name +
+ throw new IllegalArgumentException("Unrecognized column type for column: " +
+ getName(StandardCharsets.US_ASCII) +
". I regret I don't recognize: " + (char) type);
}
}
[2/2] tika git commit: TIKA-1837 -- strip comments before trying to
find encoding in HTMLEncodingDetector
Posted by ta...@apache.org.
TIKA-1837 -- strip comments before trying to find encoding in HTMLEncodingDetector
Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/16290d86
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/16290d86
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/16290d86
Branch: refs/heads/master
Commit: 16290d86c3a976f7c323b0be6750ad07f4840018
Parents: 1a04f80
Author: tballison <ta...@mitre.org>
Authored: Thu May 26 10:50:02 2016 -0400
Committer: tballison <ta...@mitre.org>
Committed: Thu May 26 10:50:02 2016 -0400
----------------------------------------------------------------------
.../tika/parser/html/HtmlEncodingDetector.java | 16 +++++++++++++-
.../apache/tika/parser/html/HtmlParserTest.java | 22 +++++++++++++++++++-
2 files changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tika/blob/16290d86/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
index edb014b..1ebe076 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlEncodingDetector.java
@@ -83,8 +83,22 @@ public class HtmlEncodingDetector implements EncodingDetector {
// a possible character encoding hint
String head = ASCII.decode(ByteBuffer.wrap(buffer, 0, n)).toString();
+ //strip out comments
+ String headNoComments = head.replaceAll("<!--.*?(-->|$)", " ");
+ //try to find the encoding in head without comments
+ Charset charset = findCharset(headNoComments);
+ //if nothing is found, back off to find any encoding
+ if (charset == null) {
+ return findCharset(head);
+ }
+ return charset;
+
+ }
+
+ //returns null if no charset was found
+ private Charset findCharset(String s) {
- Matcher equiv = HTTP_META_PATTERN.matcher(head);
+ Matcher equiv = HTTP_META_PATTERN.matcher(s);
Matcher charsetMatcher = FLEXIBLE_CHARSET_ATTR_PATTERN.matcher("");
//iterate through meta tags
while (equiv.find()) {
http://git-wip-us.apache.org/repos/asf/tika/blob/16290d86/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
----------------------------------------------------------------------
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
index 654eb8a..d37394e 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
@@ -34,11 +34,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.tika.Tika;
+import org.apache.tika.TikaTest;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Geographic;
import org.apache.tika.metadata.Metadata;
@@ -57,8 +59,9 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.Locator;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.helpers.XMLReaderAdapter;
-public class HtmlParserTest {
+public class HtmlParserTest extends TikaTest {
@Test
public void testParseAscii() throws Exception {
@@ -1149,4 +1152,21 @@ public class HtmlParserTest {
assertEquals(1, links.size());
assertEquals(url, links.get(0));
}
+
+ @Test
+ public void testSkippingCommentsInEncodingDetection() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 10000; i++) {
+ sb.append(" ");
+ }
+ byte[] bytes = new String("<html><head>" +
+ "<!--<meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\"> -->\n" +
+ " <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />"+
+ "</head>"+sb.toString()+
+ "<body>"+
+ "\u6709\u4ec0\u4e48\u9700\u8981\u6211\u5e2e\u4f60\u7684" +
+ "</body></html>").getBytes(StandardCharsets.UTF_8);
+ XMLResult r = getXML(new ByteArrayInputStream(bytes), new AutoDetectParser(), new Metadata());
+ assertContains("\u6709\u4ec0\u4e48\u9700\u8981\u6211\u5e2e\u4f60\u7684", r.xml);
+ }
}