You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/05/29 18:27:48 UTC
[tika] 01/02: TIKA-2100 extract content language from html lang
attribute
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 8d26096e9d579bee74ac01df9ae773e66e0bfc74
Author: Gérard Bouchar <gb...@protonmail.com>
AuthorDate: Fri May 25 14:14:06 2018 +0200
TIKA-2100 extract content language from html lang attribute
---
.../org/apache/tika/sax/XHTMLContentHandler.java | 7 +-
.../org/apache/tika/parser/html/HtmlHandler.java | 27 ++++----
.../apache/tika/parser/html/HtmlParserTest.java | 76 ++++++++++------------
3 files changed, 54 insertions(+), 56 deletions(-)
diff --git a/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java
index 4742339..a200820 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java
@@ -138,7 +138,12 @@ public class XHTMLContentHandler extends SafeContentHandler {
// Call directly, so we don't go through our startElement(), which will
// ignore these elements.
- super.startElement(XHTML, "html", "html", EMPTY_ATTRIBUTES);
+ AttributesImpl htmlAttrs = new AttributesImpl();
+ String lang = metadata.get(Metadata.CONTENT_LANGUAGE);
+ if (lang != null) {
+ htmlAttrs.addAttribute("", "lang", "lang", "CDATA", lang);
+ }
+ super.startElement(XHTML, "html", "html", htmlAttrs);
newline();
super.startElement(XHTML, "head", "head", EMPTY_ATTRIBUTES);
newline();
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
index e453db1..ab3af55 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
@@ -16,20 +16,6 @@
*/
package org.apache.tika.parser.html;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.metadata.HTML;
@@ -47,6 +33,16 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
class HtmlHandler extends TextContentHandler {
// List of attributes that need to be resolved.
@@ -119,6 +115,9 @@ class HtmlHandler extends TextContentHandler {
String uri, String local, String name, Attributes atts)
throws SAXException {
+ if ("HTML".equals(name) && atts.getValue("lang") != null) {
+ metadata.set(Metadata.CONTENT_LANGUAGE, atts.getValue("lang"));
+ }
if ("SCRIPT".equals(name)) {
scriptLevel++;
}
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
index f4e630b..c92a97c 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
@@ -16,45 +16,6 @@
*/
package org.apache.tika.parser.html;
-import static java.nio.charset.StandardCharsets.ISO_8859_1;
-import static java.nio.charset.StandardCharsets.US_ASCII;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.regex.Pattern;
-
-import org.apache.commons.codec.binary.Base64;
import org.apache.tika.Tika;
import org.apache.tika.TikaTest;
import org.apache.tika.config.ServiceLoader;
@@ -62,7 +23,6 @@ import org.apache.tika.config.TikaConfig;
import org.apache.tika.detect.AutoDetectReader;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.IOUtils;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Geographic;
import org.apache.tika.metadata.Metadata;
@@ -70,7 +30,6 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.LinkContentHandler;
@@ -85,6 +44,22 @@ import org.xml.sax.Locator;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import java.io.*;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.regex.Pattern;
+
+import static java.nio.charset.StandardCharsets.*;
+import static org.junit.Assert.*;
+
public class HtmlParserTest extends TikaTest {
@Test
@@ -874,6 +849,25 @@ public class HtmlParserTest extends TikaTest {
}
/**
+ * Test case for Tika-2100
+ * @see <a href="https://issues.apache.org/jira/browse/TIKA-2100">TIKA-2100</a>
+ */
+ @Test
+ public void testHtmlLanguage() throws Exception {
+ final String html = "<html lang=\"fr\"></html>";
+
+ StringWriter sw = new StringWriter();
+ Metadata metadata = new Metadata();
+ new HtmlParser().parse(
+ new ByteArrayInputStream(html.getBytes(UTF_8)),
+ makeHtmlTransformer(sw), metadata, new ParseContext());
+
+ assertEquals("fr", metadata.get(Metadata.CONTENT_LANGUAGE));
+ assertTrue("Missing HTML lang attribute",
+ Pattern.matches("(?s)<html[^>]* lang=\"fr\".*", sw.toString()));
+ }
+
+ /**
* Test case for TIKA-961
*
* @see <a href="https://issues.apache.org/jira/browse/TIKA-961">TIKA-961</a>
--
To stop receiving notification emails like this one, please contact
tallison@apache.org.