You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/05/29 18:27:48 UTC

[tika] 01/02: TIKA-2100 extract content language from html lang attribute

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 8d26096e9d579bee74ac01df9ae773e66e0bfc74
Author: Gérard Bouchar <gb...@protonmail.com>
AuthorDate: Fri May 25 14:14:06 2018 +0200

    TIKA-2100 extract content language from html lang attribute
---
 .../org/apache/tika/sax/XHTMLContentHandler.java   |  7 +-
 .../org/apache/tika/parser/html/HtmlHandler.java   | 27 ++++----
 .../apache/tika/parser/html/HtmlParserTest.java    | 76 ++++++++++------------
 3 files changed, 54 insertions(+), 56 deletions(-)

diff --git a/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java
index 4742339..a200820 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/XHTMLContentHandler.java
@@ -138,7 +138,12 @@ public class XHTMLContentHandler extends SafeContentHandler {
             
             // Call directly, so we don't go through our startElement(), which will
             // ignore these elements.
-            super.startElement(XHTML, "html", "html", EMPTY_ATTRIBUTES);
+            AttributesImpl htmlAttrs = new AttributesImpl();
+            String lang = metadata.get(Metadata.CONTENT_LANGUAGE);
+            if (lang != null) {
+                htmlAttrs.addAttribute("", "lang", "lang", "CDATA", lang);
+            }
+            super.startElement(XHTML, "html", "html", htmlAttrs);
             newline();
             super.startElement(XHTML, "head", "head", EMPTY_ATTRIBUTES);
             newline();
diff --git a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
index e453db1..ab3af55 100644
--- a/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
+++ b/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlHandler.java
@@ -16,20 +16,6 @@
  */
 package org.apache.tika.parser.html;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import org.apache.tika.extractor.EmbeddedDocumentExtractor;
 import org.apache.tika.extractor.EmbeddedDocumentUtil;
 import org.apache.tika.metadata.HTML;
@@ -47,6 +33,16 @@ import org.xml.sax.ContentHandler;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.AttributesImpl;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 class HtmlHandler extends TextContentHandler {
 
     // List of attributes that need to be resolved.
@@ -119,6 +115,9 @@ class HtmlHandler extends TextContentHandler {
             String uri, String local, String name, Attributes atts)
             throws SAXException {
 
+        if ("HTML".equals(name) && atts.getValue("lang") != null) {
+            metadata.set(Metadata.CONTENT_LANGUAGE, atts.getValue("lang"));
+        }
         if ("SCRIPT".equals(name)) {
             scriptLevel++;
         }
diff --git a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
index f4e630b..c92a97c 100644
--- a/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
+++ b/tika-parsers/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
@@ -16,45 +16,6 @@
  */
 package org.apache.tika.parser.html;
 
-import static java.nio.charset.StandardCharsets.ISO_8859_1;
-import static java.nio.charset.StandardCharsets.US_ASCII;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.regex.Pattern;
-
-import org.apache.commons.codec.binary.Base64;
 import org.apache.tika.Tika;
 import org.apache.tika.TikaTest;
 import org.apache.tika.config.ServiceLoader;
@@ -62,7 +23,6 @@ import org.apache.tika.config.TikaConfig;
 import org.apache.tika.detect.AutoDetectReader;
 import org.apache.tika.detect.EncodingDetector;
 import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.IOUtils;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Geographic;
 import org.apache.tika.metadata.Metadata;
@@ -70,7 +30,6 @@ import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
-import org.apache.tika.parser.RecursiveParserWrapper;
 import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
 import org.apache.tika.sax.BodyContentHandler;
 import org.apache.tika.sax.LinkContentHandler;
@@ -85,6 +44,22 @@ import org.xml.sax.Locator;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
 
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import java.io.*;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.regex.Pattern;
+
+import static java.nio.charset.StandardCharsets.*;
+import static org.junit.Assert.*;
+
 public class HtmlParserTest extends TikaTest {
 
     @Test
@@ -874,6 +849,25 @@ public class HtmlParserTest extends TikaTest {
     }
 
     /**
+     * Test case for Tika-2100
+     * @see <a href="https://issues.apache.org/jira/browse/TIKA-2100">TIKA-2100</a>
+     */
+    @Test
+    public void testHtmlLanguage() throws Exception {
+        final String html = "<html lang=\"fr\"></html>";
+
+        StringWriter sw = new StringWriter();
+        Metadata metadata = new Metadata();
+        new HtmlParser().parse(
+                new ByteArrayInputStream(html.getBytes(UTF_8)),
+                makeHtmlTransformer(sw), metadata, new ParseContext());
+
+        assertEquals("fr", metadata.get(Metadata.CONTENT_LANGUAGE));
+        assertTrue("Missing HTML lang attribute",
+                Pattern.matches("(?s)<html[^>]* lang=\"fr\".*", sw.toString()));
+    }
+
+    /**
      * Test case for TIKA-961
      *
      * @see <a href="https://issues.apache.org/jira/browse/TIKA-961">TIKA-961</a>

-- 
To stop receiving notification emails like this one, please contact
tallison@apache.org.