You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2025/05/21 19:30:39 UTC
(tika) branch branch_3x updated: TIKA-4419 -- experiment with custom list of self-closeable-tags.txt (#2216)
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_3x by this push:
new 369a94097 TIKA-4419 -- experiment with custom list of self-closeable-tags.txt (#2216)
369a94097 is described below
commit 369a94097d66a7e8ae442e800f8581ac3af3dd7f
Author: Tim Allison <ta...@apache.org>
AuthorDate: Wed May 21 15:30:35 2025 -0400
TIKA-4419 -- experiment with custom list of self-closeable-tags.txt (#2216)
---
.../org/apache/tika/parser/html/JSoupParser.java | 29 +++++++++++-
.../tika/parser/html/self-closeable-tags.txt | 52 ++++++++++++++++++++++
2 files changed, 79 insertions(+), 2 deletions(-)
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java
index c34a496db..bedbce788 100644
--- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/JSoupParser.java
@@ -16,8 +16,10 @@
*/
package org.apache.tika.parser.html;
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -34,6 +36,9 @@ import org.jsoup.nodes.DataNode;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
+import org.jsoup.parser.Parser;
+import org.jsoup.parser.Tag;
+import org.jsoup.parser.TagSet;
import org.jsoup.select.NodeFilter;
import org.jsoup.select.NodeTraversor;
import org.xml.sax.ContentHandler;
@@ -70,6 +75,26 @@ public class JSoupParser extends AbstractEncodingDetectorParser {
private static final Set<MediaType> SUPPORTED_TYPES = Collections.unmodifiableSet(
new HashSet<MediaType>(Arrays.asList(MediaType.text("html"), XHTML, WAP_XHTML, X_ASP)));
+ private static final TagSet SELF_CLOSEABLE_TAGS = TagSet.Html();
+
+ static {
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(
+ JSoupParser.class.getResourceAsStream("self-closeable-tags.txt"), StandardCharsets.UTF_8))) {
+ String line = reader.readLine();
+ while (line != null) {
+ if (line.startsWith("#") || line.trim().isEmpty()) {
+ line = reader.readLine();
+ continue;
+ }
+ Tag t = SELF_CLOSEABLE_TAGS.valueOf(line.trim(), Parser.NamespaceHtml);
+ t.set(Tag.SelfClose);
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Can't find self-closeable-tags.txt");
+ }
+ }
+
@Field
private boolean extractScripts = false;
@@ -128,7 +153,7 @@ public class JSoupParser extends AbstractEncodingDetectorParser {
HtmlMapper mapper = context.get(HtmlMapper.class, new DefaultHtmlMapper());
//do better with baseUri?
- Document document = Jsoup.parse(CloseShieldInputStream.wrap(stream), charset.name(), "");
+ Document document = Jsoup.parse(CloseShieldInputStream.wrap(stream), charset.name(), "", Parser.htmlParser().tagSet(SELF_CLOSEABLE_TAGS));
document.quirksMode(Document.QuirksMode.quirks);
ContentHandler xhtml = new XHTMLDowngradeHandler(
new HtmlHandler(mapper, handler, metadata, context, extractScripts));
@@ -147,7 +172,7 @@ public class JSoupParser extends AbstractEncodingDetectorParser {
HtmlMapper mapper = context.get(HtmlMapper.class, new DefaultHtmlMapper());
//do better with baseUri?
- Document document = Jsoup.parse(html);
+ Document document = Jsoup.parse(html, Parser.htmlParser().tagSet(SELF_CLOSEABLE_TAGS));
document.quirksMode(Document.QuirksMode.quirks);
ContentHandler xhtml = new XHTMLDowngradeHandler(
new HtmlHandler(mapper, handler, metadata, context, extractScripts));
diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/org/apache/tika/parser/html/self-closeable-tags.txt b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/org/apache/tika/parser/html/self-closeable-tags.txt
new file mode 100644
index 000000000..ff44a4385
--- /dev/null
+++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/resources/org/apache/tika/parser/html/self-closeable-tags.txt
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# See TIKA-4419
+br
+img
+meta
+input
+link
+path
+a
+hr
+image
+col
+area
+param
+base
+td
+div
+p
+source
+option
+embed
+span
+tr
+td
+script
+wbr
+li
+button
+iframe
+b
+i
+ul
+form
+textarea
+table
+select
+strong
+title
\ No newline at end of file