You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2026/07/06 16:57:38 UTC

(allura) branch db/remove_html5lib_turbohtml created (now acc360c76)

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a change to branch db/remove_html5lib_turbohtml
in repository https://gitbox.apache.org/repos/asf/allura.git


      at acc360c76 POC: Using sanitize/clean instead of tree walking, but now necessitates placeholder logic for unsupported logic

This branch includes the following new commits:

     new 14bfcac21 html5lib is removed with turbohtml as the replacement everywhere it was used.
     new a96316dfa disallowed tags are now preserved as escaped visible text instead of disappearing.
     new acc360c76 POC: Using sanitize/clean instead of tree walking, but now necessitates placeholder logic for unsupported logic

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(allura) 03/03: POC: Using sanitize/clean instead of tree walking, but now necessitates placeholder logic for unsupported logic

Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch db/remove_html5lib_turbohtml
in repository https://gitbox.apache.org/repos/asf/allura.git

commit acc360c76815f84ad75ce446f0936aef09a994df
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Mon Jul 6 12:53:17 2026 -0400

    POC: Using sanitize/clean instead of tree walking, but now necessitates placeholder logic for unsupported logic
    
    Allura/allura/lib/sanitize_placeholder_poc.py — PlaceholderForgeHTMLSanitizer, subclassing ForgeHTMLSanitizer so all whitelist data and helpers are reused. The flow:
    
    1. Pre-pass: parse, find iframes/inputs; valid ones (YouTube src prefix / type=checkbox) get their attributes whitelisted by our existing _sanitize_attrs (they bypass the library, so e.g. onclick/onload must die here), iframes get clear()ed, and each is swapped for a <span data-allura-placeholder="{nonce}-{i}">. The nonce is a fresh secrets.token_hex(8) per call, so user content can't forge a working placeholder — a forged one just becomes a plain <span>, content intact. Invalid ifram [...]
    2. turbohtml.clean.Sanitizer does everything else: element/attribute whitelisting, escape-with-attributes of disallowed tags, URL scheme checks, CSS scrubbing (its css_properties replaces our whole ported sanitize_css), comment stripping. Our attribute_filter shrinks to class/id/data-URI rules.
    3. Post-pass: re-parse, restore stashed elements at their nonce-matched placeholders, add misleading-link suffixes.
    
    it deletes the ~135 lines of walker/escape/URL/CSS logic but adds back ~55 lines of placeholder machinery, and it costs 3 parses per render instead of 1 (all C, and markdown is cached, so this is minor).
    
    but the walker may be simpler to reason about than the stash/restore lifecycle
---
 Allura/allura/lib/sanitize_placeholder_poc.py | 130 ++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)

diff --git a/Allura/allura/lib/sanitize_placeholder_poc.py b/Allura/allura/lib/sanitize_placeholder_poc.py
new file mode 100644
index 000000000..bec93dc53
--- /dev/null
+++ b/Allura/allura/lib/sanitize_placeholder_poc.py
@@ -0,0 +1,130 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+"""
+Proof of concept: sanitize with turbohtml.clean's Policy-based sanitizer instead of
+our hand-rolled tree walk in ForgeHTMLSanitizer.
+
+turbohtml.clean has a non-overridable safety baseline that never allows <iframe>,
+and no way to conditionally allow an element based on its attributes (we allow
+iframes only from youtube embed URLs, and <input> only for checkboxes).  To work
+around that, valid iframes/checkbox-inputs are stashed and replaced with placeholder
+elements before sanitizing, then restored afterwards.  Placeholders carry a per-call
+random nonce so user content can't forge one.
+
+The library then handles: element/attribute whitelisting, escaping disallowed tags,
+URL scheme checks (including obfuscated javascript: forms), CSS scrubbing of style
+attributes, and comment removal.  We keep: the conditional iframe/checkbox rules
+(the placeholder dance), class/id/data-URI filtering (Policy.attribute_filter), and
+the misleading-link suffixes (a post-pass that also restores the placeholders).
+"""
+
+import logging
+import secrets
+
+from turbohtml import parse_fragment, Element, Text
+from turbohtml.clean import Policy, Sanitizer, DEFAULT_CSS_PROPERTIES
+
+from allura.lib.utils import ForgeHTMLSanitizer
+
+log = logging.getLogger(__name__)
+
+
+class PlaceholderForgeHTMLSanitizer(ForgeHTMLSanitizer):
+
+    PLACEHOLDER_ATTR = 'data-allura-placeholder'
+
+    def __init__(self):
+        super().__init__()
+        self._policy = Policy(
+            # iframe & input are excluded: invalid ones get escaped by the library,
+            # valid ones bypass it entirely via the placeholder stash
+            tags=frozenset(self.allowed_elements - {'iframe', 'input'}),
+            attributes={'*': frozenset(self.allowed_attributes | {self.PLACEHOLDER_ATTR})},
+            url_schemes=frozenset(self.allowed_url_schemes),
+            attribute_filter=self._attribute_filter,
+            # DEFAULT_CSS_PROPERTIES is html5lib's whitelist; add the shorthands our
+            # sanitize_css() allowed (the library scrubs url()/expression() in values)
+            css_properties=frozenset(DEFAULT_CSS_PROPERTIES | {'background', 'border', 'margin', 'padding'}),
+        )
+
+    def sanitize(self, html: str) -> str:
+        root = parse_fragment(html)
+        nonce = secrets.token_hex(8)
+        stash = self._stash_conditional_elements(root, nonce)
+        cleaned = Sanitizer(self._policy).sanitize(root.inner_html)
+        return self._restore_and_finish(cleaned, nonce, stash)
+
+    def _stash_conditional_elements(self, root, nonce: str) -> list[Element]:
+        """
+        Detach valid iframes/checkbox-inputs, leaving nonce-marked placeholders.
+        Invalid ones are left in place for the library to escape.
+        """
+        stash = []
+        for el in list(root.find_all(['iframe', 'input'])):
+            if el.tag == 'iframe':
+                ok = (el.attrs.get('src') or '').startswith(self.valid_iframe_srcs)
+            else:
+                ok = el.attrs.get('type') == 'checkbox'
+            if ok:
+                # stashed elements bypass the library, so whitelist their attributes ourselves
+                self._sanitize_attrs(el)
+                if el.tag == 'iframe':
+                    el.clear()  # iframe contents are fallback-only, and would serialize unescaped
+                placeholder = Element('span', {self.PLACEHOLDER_ATTR: f'{nonce}-{len(stash)}'})
+                el.replace_with(placeholder)
+                stash.append(el)
+        return stash
+
+    def _attribute_filter(self, tag: str, attr: str, value: str) -> str | None:
+        # scheme filtering is handled by Policy.url_schemes; this adds our extra rules
+        if attr in self._url_attributes and not self._url_allowed(value):
+            return None  # e.g. a data: URI that isn't one of the allowed image types
+        if attr == 'class':
+            classes = value.split()
+            cleaned_classes = [c for c in classes
+                               if c in self.valid_class_values or c.startswith(self.valid_partial_class_prefixes)]
+            if cleaned_classes != classes:
+                log.info(f'Removed invalid classes: {classes} => {cleaned_classes}')
+            return ' '.join(cleaned_classes)
+        if attr == 'id':
+            if not any(value.startswith(prefix) for prefix in self.valid_id_prefixes):
+                return 'user-content-' + value
+        return value
+
+    def _restore_and_finish(self, cleaned: str, nonce: str, stash: list[Element]) -> str:
+        root = parse_fragment(cleaned)
+        used = set()
+        for placeholder in list(root.find_all('span')):
+            marker = placeholder.attrs.get(self.PLACEHOLDER_ATTR)
+            if marker is None:
+                continue
+            marker_nonce, _, index_str = marker.rpartition('-')
+            index = int(index_str) if index_str.isdigit() else -1
+            if marker_nonce == nonce and 0 <= index < len(stash) and index not in used:
+                used.add(index)
+                placeholder.replace_with(stash[index])
+            else:
+                # forged placeholder from user content: keep it as a plain span
+                del placeholder.attrs[self.PLACEHOLDER_ATTR]
+
+        for link in root.find_all('a'):
+            suffix = self._link_suffix(link)
+            if suffix:
+                link.insert_after(Text(suffix))
+
+        return root.inner_html


(allura) 01/03: html5lib is removed with turbohtml as the replacement everywhere it was used.

Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch db/remove_html5lib_turbohtml
in repository https://gitbox.apache.org/repos/asf/allura.git

commit 14bfcac212d1142a661e92c069528ae289cb8186
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Sat Jul 4 15:18:40 2026 -0400

    html5lib is removed with turbohtml as the replacement everywhere it was used.
    
    Design: a tree-walk sanitizer, and why not turbohtml.clean.sanitize
    The venv's turbohtml is a git dev build (0.4.0.dev225) that includes the clean module with sanitize()/Policy — and it covers a lot (attribute filter hook, CSS scrubbing, escape/strip/remove modes). But I found its non-overridable safety baseline escapes <iframe> in every configuration — no policy setting can allow one through. Allura needs YouTube embed iframes to survive sanitization (the [[embed]] macro and hand-written embeds), so clean.sanitize can't be used without a fragile plac [...]
    
    explicit element whitelist (same effective set as the nh3 branch, plus tfoot which that branch accidentally dropped); disallowed tags are stripped keeping contents, except script/style/textarea/select/datalist which are removed entirely; SVG/MathML unwrapped
    attribute whitelist + URL-scheme checking ported from html5lib (handles jAva\tscript: obfuscation), data: URIs limited to the same image types, style through the ported html5lib CSS sanitizer
    conditional YouTube iframes (their fallback content is cleared — turbohtml serializes iframe innards verbatim as rawtext, so leaving it would be an injection vector), checkbox-only inputs, class/id filtering, and the misleading-link "(hostname)" suffix — all in the same pass
    RelativeLinkRewriter also moved from BeautifulSoup+html5lib to turbohtml (real WHATWG parsing, unlike the html.parser fallback on the nh3 branch), and its bs4 quirks went away: inter-element whitespace and attribute order are now preserved (bs4 sorted attributes alphabetically), and <br> serializes WHATWG-style instead of <br/>.
---
 Allura/allura/lib/markdown_extensions.py           |  49 +---
 Allura/allura/lib/utils.py                         | 305 +++++++++++++--------
 Allura/allura/tests/functional/test_newforge.py    |   5 +-
 Allura/allura/tests/test_globals.py                |  25 +-
 Allura/allura/tests/test_helpers.py                |   2 +-
 Allura/allura/tests/test_markdown.py               |   2 +-
 Allura/allura/tests/test_utils.py                  |  59 ++--
 Allura/allura/tests/unit/test_solr.py              |   3 +-
 .../forgetracker/tests/functional/test_root.py     |   6 +-
 requirements.in                                    |   3 +-
 requirements.txt                                   |   6 +-
 11 files changed, 249 insertions(+), 216 deletions(-)

diff --git a/Allura/allura/lib/markdown_extensions.py b/Allura/allura/lib/markdown_extensions.py
index a7b827d4a..dd18d034a 100644
--- a/Allura/allura/lib/markdown_extensions.py
+++ b/Allura/allura/lib/markdown_extensions.py
@@ -18,7 +18,6 @@
 from __future__ import annotations
 import re
 import logging
-import warnings
 import xml.etree.ElementTree as etree
 import os
 
@@ -26,10 +25,7 @@
 
 from tg import config
 from tg import tmpl_context as c
-from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
-import html5lib
-import html5lib.serializer
-import html5lib.filters.alphabeticalattributes
+from turbohtml import parse_fragment
 import markdown
 import markdown.inlinepatterns
 import markdown.util
@@ -39,7 +35,7 @@
 from . import macro
 from . import helpers as h
 from allura import model as M
-from allura.lib.utils import ForgeHTMLSanitizerFilter, is_nofollow_url
+from allura.lib.utils import ForgeHTMLSanitizer, is_nofollow_url
 
 log = logging.getLogger(__name__)
 
@@ -474,38 +470,32 @@ def __init__(self, make_absolute=False):
         self._make_absolute = make_absolute
 
     def run(self, text: str):
-        with warnings.catch_warnings():
-            # sometimes short snippets of code (especially escaped html) can trigger this
-            warnings.filterwarnings('ignore', category=MarkupResemblesLocatorWarning)
-
-            soup = BeautifulSoup(text,
-                                 'html5lib')  # 'html.parser' parser gives weird </li> behaviour with test_macro_members
+        doc = parse_fragment(text)
         if self._make_absolute:
             rewrite = self._rewrite_abs
         else:
             rewrite = self._rewrite
-        for link in soup.find_all('a'):
+        for link in doc.find_all('a'):
             rewrite(link, 'href')
-        for link in soup.find_all('img'):
+        for link in doc.find_all('img'):
             rewrite(link, 'src')
 
-        # html5lib parser adds html/head/body tags, so output <body> without its own tags
-        return str(soup.body)[len('<body>'):-len('</body>')]
+        return doc.inner_html
 
     def _rewrite(self, tag, attr):
-        val = tag.get(attr)
+        val = tag.attrs.get(attr)
         if val is None:
             return
         if ' ' in val:
             # Don't urllib.quote to avoid possible double-quoting
             # just make sure no spaces
             val = val.replace(' ', '%20')
-            tag[attr] = val
+            tag.attrs[attr] = val
         if 'markdown_syntax' in val:
-            tag['rel'] = 'nofollow'
+            tag.attrs['rel'] = 'nofollow'
         if '://' in val:
             if is_nofollow_url(val):
-                tag['rel'] = 'nofollow'
+                tag.attrs['rel'] = 'nofollow'
             return
         if val.startswith('/'):
             return
@@ -519,30 +509,19 @@ def _rewrite(self, tag, attr):
             return
         # if none of the above, assume relative to directory link
         val = os.path.join('.', val)
-        tag[attr] = val
+        tag.attrs[attr] = val
 
     def _rewrite_abs(self, tag, attr):
         self._rewrite(tag, attr)
-        val = tag.get(attr)
+        val = tag.attrs.get(attr)
         val = urljoin(config['base_url'], val)
-        tag[attr] = val
+        tag.attrs[attr] = val
 
 
 class HTMLSanitizer(markdown.postprocessors.Postprocessor):
 
     def run(self, text):
-        parsed = html5lib.parseFragment(text)
-
-        # if we didn't have to customize our sanitization, could just do:
-        # return html5lib.serialize(parsed, sanitize=True)
-
-        # instead we do the same steps as that function,
-        # but add our ForgeHTMLSanitizerFilter instead of sanitize=True which would use the standard one
-        TreeWalker = html5lib.treewalkers.getTreeWalker("etree")
-        walker = TreeWalker(parsed)
-        walker = ForgeHTMLSanitizerFilter(walker)  # this is our custom step
-        s = html5lib.serializer.HTMLSerializer()
-        return s.render(walker)
+        return ForgeHTMLSanitizer().sanitize(text)
 
 
 class AutolinkPattern(markdown.inlinepatterns.InlineProcessor):
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index 8a800994d..3811efd0d 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -58,7 +58,7 @@
 from webob import exc
 from pygments.formatters import HtmlFormatter
 from setproctitle import getproctitle
-import html5lib.filters.sanitizer
+from turbohtml import parse_fragment, Element, Text, Namespace
 from ew import jinja2_ew as ew
 from ming.utils import LazyProperty
 from ming.odm.odmsession import ODMCursor
@@ -546,38 +546,102 @@ def serve_file(fp, filename, content_type, last_modified=None,
         return iter(lambda: fp.read(block_size), b'')
 
 
-class ForgeHTMLSanitizerFilter(html5lib.filters.sanitizer.Filter):
+ALLOWED_CSS_PROPERTIES = frozenset([
+    # from html5lib's sanitizer
+    'azimuth', 'background-color', 'border-bottom-color', 'border-collapse', 'border-color', 'border-left-color',
+    'border-right-color', 'border-top-color', 'clear', 'color', 'cursor', 'direction', 'display', 'elevation',
+    'float', 'font', 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', 'height',
+    'letter-spacing', 'line-height', 'overflow', 'pause', 'pause-after', 'pause-before', 'pitch', 'pitch-range',
+    'richness', 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', 'speech-rate', 'stress',
+    'text-align', 'text-decoration', 'text-indent', 'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
+    'white-space', 'width',
+])
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # remove some elements from the sanitizer whitelist
-        # <form> and <input> could be used for a social engineering attack to construct a form
-        # others are just unexpected and confusing, and have no need to be used in markdown
-        ns_html = html5lib.constants.namespaces['html']
-        _form_elements = {(ns_html, 'button'),
-                          (ns_html, 'datalist'),
-                          (ns_html, 'fieldset'),
-                          (ns_html, 'form'),
-                          (ns_html, 'input'),
-                          (ns_html, 'label'),
-                          (ns_html, 'legend'),
-                          (ns_html, 'meter'),
-                          (ns_html, 'optgroup'),
-                          (ns_html, 'option'),
-                          (ns_html, 'output'),
-                          (ns_html, 'progress'),
-                          (ns_html, 'select'),
-                          (ns_html, 'textarea'),
-                          }
-        _extra_allowed_elements = {
-            (ns_html, 'summary'),
-        }
-        self.allowed_elements = (set(html5lib.filters.sanitizer.allowed_elements) | _extra_allowed_elements) - _form_elements
-
-        # srcset is used in our own project_list/project_summary widgets
-        # which are used as macros so go through markdown
-        self.allowed_attributes = (html5lib.filters.sanitizer.allowed_attributes | {(None, 'srcset')})
+ALLOWED_CSS_KEYWORDS = frozenset([
+    'auto', 'aqua', 'black', 'block', 'blue', 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed',
+    'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', 'lime', 'maroon', 'medium', 'none',
+    'navy', 'normal', 'nowrap', 'olive', 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top',
+    'transparent', 'underline', 'white', 'yellow',
+])
+
+
+def sanitize_css(style: str) -> str:
+    # ported from html5lib's sanitizer (MIT licensed)
+    # disallow urls
+    style = re.compile(r'url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style)
+
+    # gauntlet
+    if not re.match(r"""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style):
+        return ''
+    if not re.match(r"^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style):
+        return ''
+
+    clean = []
+    for prop, value in re.findall(r"([-\w]+)\s*:\s*([^:;]*)", style):
+        if not value:
+            continue
+        if prop.lower() in ALLOWED_CSS_PROPERTIES:
+            clean.append(prop + ': ' + value + ';')
+        elif prop.split('-')[0].lower() in ['background', 'border', 'margin', 'padding']:
+            for keyword in value.split():
+                if keyword not in ALLOWED_CSS_KEYWORDS and \
+                        not re.match(r"^(#[0-9a-fA-F]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
+                    break
+            else:
+                clean.append(prop + ': ' + value + ';')
+
+    return ' '.join(clean)
 
+
+class ForgeHTMLSanitizer:
+    """
+    Sanitize HTML by walking the turbohtml parse tree, keeping only whitelisted
+    elements and attributes, with some additional Allura-specific rules.
+    """
+
+    # like our previous html5lib-based whitelist, minus form elements which could be
+    # used for a social engineering attack to construct a form, minus obsolete/obscure
+    # elements, and without SVG/MathML elements
+    # 'iframe' and 'input' are conditionally allowed, enforced in _sanitize_element()
+    allowed_elements = frozenset([
+        'a', 'abbr', 'acronym', 'address', 'area', 'article', 'aside', 'audio', 'b', 'bdi', 'bdo', 'big',
+        'blockquote', 'br', 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'data', 'dd', 'del',
+        'details', 'dfn', 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'figcaption', 'figure', 'font', 'footer', 'h1',
+        'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'i', 'iframe', 'img', 'input', 'ins', 'kbd', 'li',
+        'map', 'mark', 'menu', 'nav', 'ol', 'p', 'pre', 'q', 'rp', 'rt', 'rtc', 'ruby', 's', 'samp', 'section',
+        'small', 'source', 'span', 'strike', 'strong', 'sub', 'summary', 'sup', 'table', 'tbody', 'td', 'tfoot',
+        'th', 'thead', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'video', 'wbr',
+    ])
+
+    # disallowed elements whose contents aren't meaningful on their own, so are removed entirely
+    # (other disallowed elements are removed but their contents are kept)
+    _removed_elements = frozenset(['script', 'style', 'textarea', 'select', 'datalist'])
+
+    # generic attributes allowed on any element, similar to our previous html5lib-based whitelist.
+    # srcset is used in our own project_list/project_summary widgets
+    # which are used as macros so go through markdown
+    allowed_attributes = frozenset([
+        'abbr', 'accesskey', 'align', 'alt', 'axis', 'bgcolor', 'border', 'cellpadding', 'cellspacing',
+        'char', 'charoff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', 'cols', 'colspan',
+        'compact', 'controls', 'coords', 'datetime', 'dir', 'disabled', 'face', 'frame', 'headers',
+        'height', 'high', 'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang', 'longdesc',
+        'loop', 'low', 'max', 'media', 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open',
+        'optimum', 'poster', 'preload', 'prompt', 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope',
+        'shape', 'size', 'span', 'src', 'srcset', 'start', 'style', 'summary', 'tabindex', 'target',
+        'title', 'type', 'usemap', 'valign', 'value', 'vspace', 'width', 'wrap',
+    ])
+
+    # attributes whose value is a URL, checked against allowed_url_schemes
+    _url_attributes = frozenset(['href', 'src', 'cite', 'longdesc', 'poster', 'usemap'])
+
+    # 'data' URIs are further restricted to a few image types in _url_allowed()
+    allowed_url_schemes = frozenset([
+        'afs', 'aim', 'bitcoin', 'callto', 'data', 'ed2k', 'feed', 'ftp', 'ftps', 'geo', 'gopher', 'http', 'https',
+        'im', 'irc', 'ircs', 'magnet', 'mailto', 'mms', 'mx', 'news', 'nntp', 'openpgp4fpr', 'rsync', 'rtsp',
+        'sftp', 'sip', 'sms', 'smsto', 'ssh', 'tag', 'tel', 'telnet', 'url', 'urn', 'webcal', 'wtai', 'xmpp',
+    ])
+
+    def __init__(self):
         self.valid_iframe_srcs = ('https://www.youtube.com/embed/',
                                   'https://www.youtube-nocookie.com/embed/',
                                   )
@@ -606,103 +670,105 @@ def __init__(self, *args, **kwargs):
             'h-',  # see toc_slugify_with_prefix
             'fn:', 'fnref:',  # from footnotes extension
         } | set(aslist(tg.config.get('safe_html.id_prefixes', [])))
-        self._prev_token_was_ok_iframe = False
-        self._current_link = None
-        self._pending_link_suffix = None
-
-    def __iter__(self):
-        # override to inject link suffixes in the output
-        for token in super().__iter__():
-            yield token
-            if self._pending_link_suffix is not None:
-                yield self._pending_link_suffix
-                self._pending_link_suffix = None
-
-    def sanitize_token(self, token):
-        """
-        Allow iframe tags if the src attribute matches our list of valid sources.
-        Allow input tags if the type attribute matches "checkbox"
-        Otherwise use default sanitization.
-        """
-
-        iframe_el = (html5lib.constants.namespaces['html'], 'iframe')
-        self.allowed_elements.discard(iframe_el)
-        ok_opening_iframe = False
-
-        input_el = (html5lib.constants.namespaces['html'], 'input')
-        self.allowed_elements.discard(input_el)
-
-        if token.get('name') == 'iframe':
-            attrs = token.get('data') or {}
-            if attrs.get((None, 'src'), '').startswith(self.valid_iframe_srcs):
-                self.allowed_elements.add(iframe_el)
-                ok_opening_iframe = True
-            elif token.get('type') == "EndTag" and self._prev_token_was_ok_iframe:
-                self.allowed_elements.add(iframe_el)
-
-        # sanitize classes and ids
-        if token.get('type') == 'StartTag':
-            classes = token.get('data', {}).get((None, 'class'), '')
-            if classes:
-                classes = classes.split()
-                cleaned_classes = [c for c in classes if c in self.valid_class_values or c.startswith(self.valid_partial_class_prefixes)]
-                if cleaned_classes != classes:
-                    log.info(f'Removed invalid classes: {classes} => {cleaned_classes}')
-                token['data'][(None, 'class')] = ' '.join(cleaned_classes)
 
-            id = token.get('data', {}).get((None, 'id'), '')
-            if id:
-                if not any(id.startswith(prefix) for prefix in self.valid_id_prefixes):
-                    token['data'][(None, 'id')] = 'user-content-' + id
+    def sanitize(self, html: str) -> str:
+        root = parse_fragment(html)
+        self._sanitize_children(root)
+        return root.inner_html
 
-        self._prev_token_was_ok_iframe = ok_opening_iframe
+    def _sanitize_children(self, parent) -> None:
+        for child in list(parent.children):
+            if isinstance(child, Element):
+                self._sanitize_element(child)
+            elif not isinstance(child, Text):
+                child.decompose()  # comments etc.
 
-        if token.get('name') == 'input':
-            attrs = token.get('data') or {}
-            if attrs.get((None, 'type'), '') == "checkbox":
-                self.allowed_elements.add(input_el)
-
-        sanitized = super().sanitize_token(token)
-        self._track_link_target(sanitized)
-        return sanitized
-
-    def _track_link_target(self, token):
-        if token is None:
+    def _sanitize_element(self, el: Element) -> None:
+        """
+        Keep whitelisted elements, and additionally:
+        allow iframe tags only if the src attribute matches our list of valid sources
+        allow input tags only if the type attribute matches "checkbox"
+        append a "(hostname)" suffix after links with misleading visible text
+        """
+        tag = el.tag
+        if tag in self._removed_elements:
+            el.decompose()
             return
-
-        tag_type = token.get('type')
-        tag_name = token.get('name')
-
-        if tag_type == 'StartTag' and tag_name == 'a':
-            href = token.get('data', {}).get((None, 'href'))
-            hostname = hostname_from_url(href)
-            self._current_link = {
-                'href': href,
-                'hostname': hostname,
-                'text_parts': [],
-            }
+        if el.namespace is not Namespace.HTML or tag not in self.allowed_elements:
+            # drop the tag itself but keep its (sanitized) contents
+            self._sanitize_children(el)
+            el.unwrap()
             return
 
-        if self._current_link is None:
-            return
+        self._sanitize_attrs(el)
 
-        if tag_type in ('Characters', 'SpaceCharacters'):
-            self._current_link['text_parts'].append(token.get('data', ''))
+        if tag == 'iframe':
+            if (el.attrs.get('src') or '').startswith(self.valid_iframe_srcs):
+                el.clear()  # iframe contents are fallback-only, and would serialize unescaped
+            else:
+                el.decompose()
+            return
+        if tag == 'input':
+            if el.attrs.get('type') != 'checkbox':
+                el.decompose()
             return
 
-        if tag_type == 'EndTag' and tag_name == 'a':
-            self._finalize_link_target()
-            self._current_link = None
+        self._sanitize_children(el)
+
+        if tag == 'a':
+            suffix = self._link_suffix(el)
+            if suffix:
+                el.insert_after(Text(suffix))
+
+    def _sanitize_attrs(self, el: Element) -> None:
+        for name in list(el.attrs):
+            if name not in self.allowed_attributes:
+                del el.attrs[name]
+                continue
+            value = el.attrs[name]
+            if value is None:
+                continue
+            if isinstance(value, list):
+                value = ' '.join(value)
+            if name in self._url_attributes and not self._url_allowed(value):
+                del el.attrs[name]
+            elif name == 'class':
+                classes = value.split()
+                cleaned_classes = [c for c in classes
+                                   if c in self.valid_class_values or c.startswith(self.valid_partial_class_prefixes)]
+                if cleaned_classes != classes:
+                    log.info(f'Removed invalid classes: {classes} => {cleaned_classes}')
+                el.attrs['class'] = cleaned_classes
+            elif name == 'id':
+                if not any(value.startswith(prefix) for prefix in self.valid_id_prefixes):
+                    el.attrs['id'] = 'user-content-' + value
+            elif name == 'style':
+                el.attrs['style'] = sanitize_css(value)
+
+    def _url_allowed(self, value: str) -> bool:
+        # strip chars that browsers may ignore within URLs, before detecting the scheme
+        # (same as html5lib's sanitizer did)
+        val = re.sub(r"[`\x00-\x20\x7f-\xa0\s]+", '', value).replace('�', '').lower()
+        scheme_match = re.match(r'^([a-z0-9][-+.a-z0-9]*):', val)
+        if not scheme_match:
+            return True  # relative url, no scheme
+        scheme = scheme_match.group(1)
+        if scheme not in self.allowed_url_schemes:
+            return False
+        if scheme == 'data':
+            # only allow data URIs for a few image types (and text/plain)
+            return bool(re.match(r'^data:(image/(png|jpeg|gif|webp|bmp)|text/plain)[;,]', val))
+        return True
 
-    def _finalize_link_target(self):
-        href = self._current_link['href']
-        hostname = self._current_link['hostname']
+    def _link_suffix(self, link: Element) -> str | None:
+        href = link.attrs.get('href')
+        hostname = hostname_from_url(href)
         if not href or not hostname:
-            return
+            return None
 
-        visible_text = ''.join(self._current_link['text_parts']).strip()
+        visible_text = link.text.strip()
         if not visible_text:
-            return
+            return None
 
         idn_lookalike = 'xn--' in hostname and idn_uses_lookalike_chars(hostname)
         if not idn_lookalike:
@@ -710,14 +776,11 @@ def _finalize_link_target(self):
             # gets the suffix so the deception is visible; all other links only get it
             # when the visible text looks like a URL pointing somewhere different.
             if not text_contains_any_url(visible_text):
-                return
+                return None
             if text_contains_hostname(visible_text, hostname):
-                return
+                return None
 
-        self._pending_link_suffix = {
-            'type': 'Characters',
-            'data': f' ({hostname})',
-        }
+        return f' ({hostname})'
 
 
 # Scripts whose characters are visually distinct from ASCII and unlikely to be used
diff --git a/Allura/allura/tests/functional/test_newforge.py b/Allura/allura/tests/functional/test_newforge.py
index c52fbf37d..c87e80078 100644
--- a/Allura/allura/tests/functional/test_newforge.py
+++ b/Allura/allura/tests/functional/test_newforge.py
@@ -34,12 +34,11 @@ def test_markdown_to_html(self):
             '/nf/markdown_to_html?markdown=*aaa*bb[wiki:Home]&project=test&app=wiki&neighborhood=%s' % n._id, validate_chunk=True)
         assert '<p><em>aaa</em>bb<a class="alink" href="/p/test/wiki/Home/">[wiki:Home]</a></p>' in r, r
 
-        # this happens to trigger an error
+        # this used to trigger a parsing error; now the unknown tag is simply removed
         bad_markdown = '<foo {bar}>'
         r = self.app.get('/nf/markdown_to_html?markdown=%s&project=test&app=wiki&neighborhood=%s' %
                          (quote(bad_markdown), n._id))
-        r.mustcontain('The markdown supplied could not be parsed correctly.')
-        r.mustcontain('<pre>&lt;foo {bar}&gt;</pre>')
+        r.mustcontain('<div class="markdown_content"><p></p></div>')
 
         r = self.app.get('/nf/markdown_to_html?markdown=*aaa*bb[wiki:Home]&project=test&app=wiki&neighborhood=bogus',
                          status=400)
diff --git a/Allura/allura/tests/test_globals.py b/Allura/allura/tests/test_globals.py
index ab1a09c17..65b3283a5 100644
--- a/Allura/allura/tests/test_globals.py
+++ b/Allura/allura/tests/test_globals.py
@@ -355,9 +355,10 @@ def test_macro_embed(self, oembed_fetch):
         }
         r = g.markdown_wiki.convert('[[embed url=http://www.youtube.com/watch?v=kOLpSPEA72U]]')
         assert ('<p><iframe height="270" '
-                'src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed" width="480"></iframe>'
-                '&lt;script&gt;alert(1)&lt;/script&gt;</p>' in
+                'src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed" width="480"></iframe></p>' in
                 r.replace('\n', ''))
+        assert 'script' not in r
+        assert 'alert' not in r
 
     def test_macro_embed_video_gone(self):
         # this does a real fetch
@@ -465,16 +466,16 @@ def test_markdown_basics(self):
 
         assert (
             g.markdown.convert('Multi\nLine') ==
-            '<div class="markdown_content"><p>Multi<br/>\n'
+            '<div class="markdown_content"><p>Multi<br>\n'
             'Line</p></div>')
         assert (
             g.markdown.convert('Multi\n\nLine') ==
             '<div class="markdown_content"><p>Multi</p>\n'
             '<p>Line</p></div>')
 
-        # should not raise an exception:
+        # should not raise an exception; unknown tag is removed:
         assert g.markdown.convert("<class 'foo'>") == \
-            '''<div class="markdown_content"><p>&lt;class 'foo'=""&gt;&lt;/class&gt;</p></div>'''
+            '<div class="markdown_content"><p></p></div>'
 
         assert g.markdown.convert(dedent('''\
             # Header
@@ -510,7 +511,7 @@ def test_markdown_link_references(self):
                [1]: http://url
                [2]: http://another.url "A funky title"
         ''')) == dedent('''\
-            <div class="markdown_content"><p>You can also put the <a href="http://url" rel="nofollow">link URL</a> below the current paragraph like <a href="http://another.url" rel="nofollow" title="A funky title">this</a>.</p></div>''')
+            <div class="markdown_content"><p>You can also put the <a href="http://url" rel="nofollow">link URL</a> below the current paragraph like <a href="http://another.url" title="A funky title" rel="nofollow">this</a>.</p></div>''')
 
         assert g.markdown.convert(dedent('''\
             Or you can use a [shortcut][] reference
@@ -579,7 +580,7 @@ def test_markdown_autolink(self):
         # beginning of doc
         assert '<a href=' in g.markdown.convert('http://domain.net abc')
         # beginning of a line
-        assert ('<br/>\n<a href="http://' in
+        assert ('<br>\n<a href="http://' in
                 g.markdown.convert('foobar\nhttp://domain.net abc'))
         # special characters allowed
         assert (' href="http://mst.dn/@acct:%20#target" ' in
@@ -601,7 +602,7 @@ def test_markdown_autolink_with_escape(self):
 
     def test_markdown_invalid_script(self):
         r = g.markdown.convert('<script>alert(document.cookies)</script>')
-        assert '<div class="markdown_content">&lt;script&gt;alert(document.cookies)&lt;/script&gt;\n</div>' == r
+        assert '<div class="markdown_content">\n</div>' == r
 
     def test_markdown_invalid_onerror(self):
         r = g.markdown.convert('<img src=x onerror=alert(document.cookie)>')
@@ -614,13 +615,13 @@ def test_markdown_invalid_tagslash(self):
     def test_markdown_invalid_script_in_link(self):
         r = g.markdown.convert('[xss](http://"><a onmouseover=prompt(document.domain)>xss</a>)')
         assert ('<div class="markdown_content"><p><a '
-                '''href='http://"&gt;&lt;a%20onmouseover=prompt(document.domain)&gt;xss&lt;/a' '''
+                'href="http://&quot;><a%20onmouseover=prompt(document.domain)>xss</a" '
                 'rel="nofollow">xss</a>)</p></div>' == r)
 
     def test_markdown_invalid_script_in_link2(self):
         r = g.markdown.convert('[xss](http://"><img src=x onerror=alert(document.cookie)>)')
         assert ('<div class="markdown_content"><p><a '
-                '''href='http://"&gt;&lt;img%20src=x%20onerror=alert(document.cookie)' '''
+                'href="http://&quot;><img%20src=x%20onerror=alert(document.cookie)" '
                 'rel="nofollow">xss</a>)</p></div>' == r)
 
     def test_markdown_invalid_classes_ids(self):
@@ -667,7 +668,7 @@ def do_test(size: int):
 
     def test_macro_include(self):
         r = g.markdown.convert('[[include ref=Home id=foo class=modal data-a=b]]')
-        assert '<div class="" id="user-content-foo">' in r, r
+        assert '<div id="user-content-foo" class="">' in r, r
         assert 'href="./foo"' in g.markdown.convert('[My foo](foo)')
         assert 'href="..' not in g.markdown.convert('[My foo](./foo)')
 
@@ -779,7 +780,7 @@ def test_hideawards_macro(self):
 
         with h.push_context(p_nbhd.neighborhood_project._id):
             r = g.markdown_wiki.convert('[[projects]]')
-            assert ('<div class="feature"> <a href="http://award.org" rel="nofollow" title="Winner!">'
+            assert ('<div class="feature"> <a href="http://award.org" title="Winner!" rel="nofollow">'
                     'Award short</a> </div>' in
                     squish_spaces(r))
 
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 5ea729e59..ae318e998 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -779,6 +779,6 @@ def test_querystring():
 
 
 def test_clean_html():
-    assert h.clean_html('<script>alert(1)</script>') == '&lt;script&gt;alert(1)&lt;/script&gt;'
+    assert h.clean_html('<script>alert(1)</script>') == ''
     assert h.clean_html('<b style="color: red; right: 0">ok</b>') == '<b style="color: red;">ok</b>'
     assert isinstance(h.clean_html('foo'), Markup)
diff --git a/Allura/allura/tests/test_markdown.py b/Allura/allura/tests/test_markdown.py
index 8bd41f863..c5724b39b 100644
--- a/Allura/allura/tests/test_markdown.py
+++ b/Allura/allura/tests/test_markdown.py
@@ -148,7 +148,7 @@ def test_convert(self, lookup, get_comment_slug):
 * <a href="/p/project/tool/artifact/">ticket:100</a><br>
 * <a href="/p/project/tool/artifact/#abc">comment:13:ticket:2</a><br>
 * <a href="/p/project/tool/2/tree/test.py#l3">source:test.py@2#L3</a></p>
-<p>Not *strong* or _underlined_.</div>"""
+<p>Not *strong* or _underlined_.</p></div>"""
 
         md = markdown.Markdown(
             extensions=[mde.CommitMessageExtension(app), 'markdown.extensions.nl2br'],
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index 0d88dde95..797f3d04b 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -30,9 +30,6 @@
 from pygments.lexers import get_lexer_for_filename
 import pytest
 from tg import config
-import html5lib
-import html5lib.treewalkers
-import html5lib.serializer
 
 from alluratest.controller import setup_unit_test
 
@@ -246,53 +243,45 @@ class TestHTMLSanitizer:
     def setup_method(self, method):
         setup_unit_test()
 
-    def walker_from_text(self, text):
-        parsed = html5lib.parseFragment(text)
-        TreeWalker = html5lib.treewalkers.getTreeWalker("etree")
-        walker = TreeWalker(parsed)
-        return walker
-
     def sanitize_html(self, html):
         with h.push_config(config, domain='mysite.com'):
-            filt = utils.ForgeHTMLSanitizerFilter(self.walker_from_text(html))
-            return html5lib.serializer.HTMLSerializer().render(filt)
-
-    def simple_tag_list(self, sanitizer):
-        # no attrs, no close tag flag check, just real simple
-        return [
-            t['name'] for t in sanitizer if t.get('name')
-        ]
+            return utils.ForgeHTMLSanitizer().sanitize(html)
 
     def test_html_sanitizer_iframe(self):
-        walker = self.walker_from_text('<div><iframe></iframe></div>')
-        p = utils.ForgeHTMLSanitizerFilter(walker)
-        assert self.simple_tag_list(p) == ['div', 'div']
+        assert self.sanitize_html('<div><iframe></iframe></div>') == '<div></div>'
 
     def test_html_sanitizer_youtube_iframe(self):
-        walker = self.walker_from_text(
+        assert (self.sanitize_html(
+            '<div><iframe src="https://www.youtube.com/embed/kOLpSPEA72U?feature=oembed"></iframe></div>') ==
             '<div><iframe src="https://www.youtube.com/embed/kOLpSPEA72U?feature=oembed"></iframe></div>')
-        p = utils.ForgeHTMLSanitizerFilter(walker)
-        assert self.simple_tag_list(p) == ['div', 'iframe', 'iframe', 'div']
 
-        walker = self.walker_from_text(
+        assert (self.sanitize_html(
+            '<div><iframe src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed"></iframe></div>') ==
             '<div><iframe src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed"></iframe></div>')
-        p = utils.ForgeHTMLSanitizerFilter(walker)
-        assert self.simple_tag_list(p) == ['div', 'iframe', 'iframe', 'div']
+
+        assert (self.sanitize_html('<div><iframe src="https://evil.com/embed/x"></iframe></div>') ==
+                '<div></div>')
 
     def test_html_sanitizer_form_elements(self):
-        walker = self.walker_from_text('<p>test</p><form method="post" action="http://localhost/foo.php"><input type=file><input type=text><textarea>asdf</textarea></form>')
-        p = utils.ForgeHTMLSanitizerFilter(walker)
-        assert self.simple_tag_list(p) == ['p', 'p']
+        assert (self.sanitize_html('<p>test</p><form method="post" action="http://localhost/foo.php">'
+                                   '<input type=file><input type=text><textarea>asdf</textarea></form>') ==
+                '<p>test</p>')
 
     def test_html_sanitizer_checkbox(self):
-        walker = self.walker_from_text('<p><input type="checkbox" disabled/><input type="text" disabled/><input type="checkbox" disabled checked/></p>')
-        p = utils.ForgeHTMLSanitizerFilter(walker)
-        assert self.simple_tag_list(p) == ['p', 'input', 'input', 'p']
+        assert (self.sanitize_html('<p><input type="checkbox" disabled/><input type="text" disabled/>'
+                                   '<input type="checkbox" disabled checked/></p>') ==
+                '<p><input type="checkbox" disabled=""><input type="checkbox" disabled="" checked=""></p>')
 
     def test_html_sanitizer_summary(self):
-        walker = self.walker_from_text('<details open="open"><summary>An Summary</summary><ul><li>Bullet Item</li></ul></details>')
-        p = utils.ForgeHTMLSanitizerFilter(walker)
-        assert self.simple_tag_list(p) == ['details', 'summary', 'summary', 'ul', 'li', 'li', 'ul', 'details']
+        assert (self.sanitize_html('<details open="open"><summary>An Summary</summary><ul><li>Bullet Item</li></ul></details>') ==
+                '<details open="open"><summary>An Summary</summary><ul><li>Bullet Item</li></ul></details>')
+
+    def test_html_sanitizer_dangerous_urls(self):
+        assert self.sanitize_html('<a href="javascript:alert(1)">x</a>') == '<a>x</a>'
+        assert self.sanitize_html('<a href="jAva\tscript:alert(1)">x</a>') == '<a>x</a>'
+        assert (self.sanitize_html('<img src="data:image/png;base64,AAAA">') ==
+                '<img src="data:image/png;base64,AAAA">')
+        assert self.sanitize_html('<a href="data:text/html,<script>x</script>">y</a>') == '<a>y</a>'
 
     def test_misleading_links(self):
         # OK: generic text
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
index e25581ffd..a0bdcc9a2 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -157,8 +157,9 @@ def test_solarize_strips_markdown(self):
         assert self.obj.solarize() == dict(text='Header')
 
     def test_solarize_html_in_text(self):
+        # raw script tag is removed entirely by sanitization, so not indexed (matching what is displayed)
         self.obj.index = lambda: dict(text='<script>a(1)</script>')
-        assert self.obj.solarize() == dict(text='<script>a(1)</script>')
+        assert self.obj.solarize() == dict(text='')
         self.obj.index = lambda: dict(text='&lt;script&gt;a(1)&lt;/script&gt;')
         assert self.obj.solarize() == dict(text='<script>a(1)</script>')
 
diff --git a/ForgeTracker/forgetracker/tests/functional/test_root.py b/ForgeTracker/forgetracker/tests/functional/test_root.py
index f64acd9b3..0f3857e1e 100644
--- a/ForgeTracker/forgetracker/tests/functional/test_root.py
+++ b/ForgeTracker/forgetracker/tests/functional/test_root.py
@@ -892,7 +892,7 @@ def test_delete_attachment(self):
         assert file_name in ticket_editor
         req = self.app.get('/bugs/1/')
         form = self._find_update_ticket_form(req)
-        file_link = BeautifulSoup(form.text, features="html5lib").find_all('a')[2]
+        file_link = BeautifulSoup(form.text, features="html.parser").find_all('a')[2]
         assert file_link.string == file_name
         self.app.post(str(file_link['href']), {
             'delete': 'True'
@@ -935,7 +935,7 @@ def test_new_text_attachment_content(self):
             'summary': 'zzz'
         }, upload_files=[upload]).follow()
         form = self._find_update_ticket_form(ticket_editor)
-        download = self.app.get(str(BeautifulSoup(form.text, features="html5lib").find_all('a')[2]['href']))
+        download = self.app.get(str(BeautifulSoup(form.text, features="html.parser").find_all('a')[2]['href']))
         assert download.body == file_data
 
     def test_two_attachments(self):
@@ -1315,7 +1315,7 @@ def test_new_ticket_validation(self):
         error_form = form.submit()
         form = self._find_new_ticket_form(error_form)
         assert form['ticket_form.labels'].value == 'foo'
-        error_message = BeautifulSoup(form.text, features="html5lib").find('div', {'class': 'error'})
+        error_message = BeautifulSoup(form.text, features="html.parser").find('div', {'class': 'error'})
         assert error_message
         assert (error_message.string == 'You must provide a Title' or
                 error_message.string == 'Missing value')
diff --git a/requirements.in b/requirements.in
index 43af7b8bf..c17fe734e 100644
--- a/requirements.in
+++ b/requirements.in
@@ -14,7 +14,6 @@ feedgenerator
 feedparser
 FormEncode
 GitPython
-html5lib
 idna
 Jinja2
 # Webob uses legacy-cgi and so our tests need it too for cgi.FieldStorage, maybe webob 2 will remove it?  https://github.com/Pylons/webob/pull/466
@@ -53,6 +52,8 @@ six
 sxsdiff
 TimerMiddleware
 TurboGears2
+# pre-release version, no PyPI release has all we need yet
+turbohtml @ git+https://github.com/tox-dev/turbohtml@76a6d3b1d588b93c403fc0a6238b556d09871b33
 WebHelpers2
 WebOb
 werkzeug
diff --git a/requirements.txt b/requirements.txt
index fd130d5d3..528396237 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -105,9 +105,7 @@ gunicorn==25.3.0
 h11==0.16.0
     # via httpcore
 html5lib==1.1
-    # via
-    #   -r requirements.in
-    #   pypeline
+    # via pypeline
 httpcore==1.0.9
     # via httpx
 httpx==0.27.2
@@ -319,6 +317,8 @@ translationstring==1.4
     # via colander
 turbogears2==2.5.0
     # via -r requirements.in
+turbohtml @ git+https://github.com/tox-dev/turbohtml@76a6d3b1d588b93c403fc0a6238b556d09871b33
+    # via -r requirements.in
 typing-extensions==4.15.0
     # via beautifulsoup4
 tzdata==2025.2 ; sys_platform == 'win32'


(allura) 02/03: disallowed tags are now preserved as escaped visible text instead of disappearing.

Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch db/remove_html5lib_turbohtml
in repository https://gitbox.apache.org/repos/asf/allura.git

commit a96316dfa6696cab739510489958e4852dbc9b83
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Sat Jul 4 15:31:34 2026 -0400

    disallowed tags are now preserved as escaped visible text instead of disappearing.
    
    How it works
    
    _sanitize_element() routes every disallowed element (unknown tags, form elements, script/style, SVG/MathML, non-YouTube iframes, non-checkbox inputs) through a new _escape_element(): it reconstructs the tag as Text nodes — start tag with its attributes, the already-sanitized children hoisted in place, and an end tag unless the element is a void element like <input>. The WHATWG serializer escapes those text nodes on output, so <script>alert(1)</script> renders as visible &lt;script&gt; [...]
    
    This is exactly what the old html5lib sanitizer did, so several tests I'd previously changed reverted to their original master expectations byte-for-byte: the escaped <script> test, <class 'foo'=""></class>, clean_html, solr's solarize (script text is searchable again, matching what's displayed), and the compromised-oembed test showing the escaped script after the iframe. The <foo {bar}> case that used to crash the old parser now renders as escaped text rather than the error-fallback page.
    
    One residual difference from the old sanitizer, for completeness: disallowed attributes on allowed elements are still silently dropped (both old and new behavior), and escaped-tag output always uses name="value" form — matching how html5lib rendered them.
---
 Allura/allura/lib/utils.py                      | 42 ++++++++++++++-----------
 Allura/allura/tests/functional/test_newforge.py |  4 +--
 Allura/allura/tests/test_globals.py             | 11 +++----
 Allura/allura/tests/test_helpers.py             |  2 +-
 Allura/allura/tests/test_utils.py               | 12 ++++---
 Allura/allura/tests/unit/test_solr.py           |  3 +-
 6 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index 3811efd0d..5b846ba91 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -597,8 +597,14 @@ class ForgeHTMLSanitizer:
     """
     Sanitize HTML by walking the turbohtml parse tree, keeping only whitelisted
     elements and attributes, with some additional Allura-specific rules.
+    Disallowed elements are escaped into visible text (like our previous html5lib-based
+    sanitizer did), so someone writing about e.g. script tags doesn't have them vanish.
     """
 
+    # end tags aren't written for these when escaping a disallowed element
+    _void_elements = frozenset(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
+                                'param', 'source', 'track', 'wbr'])
+
     # like our previous html5lib-based whitelist, minus form elements which could be
     # used for a social engineering attack to construct a form, minus obsolete/obscure
     # elements, and without SVG/MathML elements
@@ -613,10 +619,6 @@ class ForgeHTMLSanitizer:
         'th', 'thead', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'video', 'wbr',
     ])
 
-    # disallowed elements whose contents aren't meaningful on their own, so are removed entirely
-    # (other disallowed elements are removed but their contents are kept)
-    _removed_elements = frozenset(['script', 'style', 'textarea', 'select', 'datalist'])
-
     # generic attributes allowed on any element, similar to our previous html5lib-based whitelist.
     # srcset is used in our own project_list/project_summary widgets
     # which are used as macros so go through markdown
@@ -691,26 +693,20 @@ def _sanitize_element(self, el: Element) -> None:
         append a "(hostname)" suffix after links with misleading visible text
         """
         tag = el.tag
-        if tag in self._removed_elements:
-            el.decompose()
-            return
-        if el.namespace is not Namespace.HTML or tag not in self.allowed_elements:
-            # drop the tag itself but keep its (sanitized) contents
+        allowed = el.namespace is Namespace.HTML and tag in self.allowed_elements
+        if allowed and tag == 'iframe':
+            allowed = (el.attrs.get('src') or '').startswith(self.valid_iframe_srcs)
+        if allowed and tag == 'input':
+            allowed = el.attrs.get('type') == 'checkbox'
+        if not allowed:
             self._sanitize_children(el)
-            el.unwrap()
+            self._escape_element(el)
             return
 
         self._sanitize_attrs(el)
 
         if tag == 'iframe':
-            if (el.attrs.get('src') or '').startswith(self.valid_iframe_srcs):
-                el.clear()  # iframe contents are fallback-only, and would serialize unescaped
-            else:
-                el.decompose()
-            return
-        if tag == 'input':
-            if el.attrs.get('type') != 'checkbox':
-                el.decompose()
+            el.clear()  # iframe contents are fallback-only, and would serialize unescaped
             return
 
         self._sanitize_children(el)
@@ -720,6 +716,16 @@ def _sanitize_element(self, el: Element) -> None:
             if suffix:
                 el.insert_after(Text(suffix))
 
+    def _escape_element(self, el: Element) -> None:
+        # replace a disallowed element with visible text of its tags (escaped during serialization),
+        # keeping its already-sanitized contents in place
+        attrs = ''.join(f' {name}="{" ".join(value) if isinstance(value, list) else value or ""}"'
+                        for name, value in el.attrs.items())
+        el.insert_before(Text(f'<{el.tag}{attrs}>'))
+        if el.tag not in self._void_elements:
+            el.insert_after(Text(f'</{el.tag}>'))
+        el.unwrap()
+
     def _sanitize_attrs(self, el: Element) -> None:
         for name in list(el.attrs):
             if name not in self.allowed_attributes:
diff --git a/Allura/allura/tests/functional/test_newforge.py b/Allura/allura/tests/functional/test_newforge.py
index c87e80078..6d18a6185 100644
--- a/Allura/allura/tests/functional/test_newforge.py
+++ b/Allura/allura/tests/functional/test_newforge.py
@@ -34,11 +34,11 @@ def test_markdown_to_html(self):
             '/nf/markdown_to_html?markdown=*aaa*bb[wiki:Home]&project=test&app=wiki&neighborhood=%s' % n._id, validate_chunk=True)
         assert '<p><em>aaa</em>bb<a class="alink" href="/p/test/wiki/Home/">[wiki:Home]</a></p>' in r, r
 
-        # this used to trigger a parsing error; now the unknown tag is simply removed
+        # this used to trigger a parsing error; now the unknown tag is escaped
         bad_markdown = '<foo {bar}>'
         r = self.app.get('/nf/markdown_to_html?markdown=%s&project=test&app=wiki&neighborhood=%s' %
                          (quote(bad_markdown), n._id))
-        r.mustcontain('<div class="markdown_content"><p></p></div>')
+        r.mustcontain('&lt;foo {bar}=""&gt;&lt;/foo&gt;')
 
         r = self.app.get('/nf/markdown_to_html?markdown=*aaa*bb[wiki:Home]&project=test&app=wiki&neighborhood=bogus',
                          status=400)
diff --git a/Allura/allura/tests/test_globals.py b/Allura/allura/tests/test_globals.py
index 65b3283a5..0af320f1b 100644
--- a/Allura/allura/tests/test_globals.py
+++ b/Allura/allura/tests/test_globals.py
@@ -355,10 +355,9 @@ def test_macro_embed(self, oembed_fetch):
         }
         r = g.markdown_wiki.convert('[[embed url=http://www.youtube.com/watch?v=kOLpSPEA72U]]')
         assert ('<p><iframe height="270" '
-                'src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed" width="480"></iframe></p>' in
+                'src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed" width="480"></iframe>'
+                '&lt;script&gt;alert(1)&lt;/script&gt;</p>' in
                 r.replace('\n', ''))
-        assert 'script' not in r
-        assert 'alert' not in r
 
     def test_macro_embed_video_gone(self):
         # this does a real fetch
@@ -473,9 +472,9 @@ def test_markdown_basics(self):
             '<div class="markdown_content"><p>Multi</p>\n'
             '<p>Line</p></div>')
 
-        # should not raise an exception; unknown tag is removed:
+        # should not raise an exception:
         assert g.markdown.convert("<class 'foo'>") == \
-            '<div class="markdown_content"><p></p></div>'
+            '''<div class="markdown_content"><p>&lt;class 'foo'=""&gt;&lt;/class&gt;</p></div>'''
 
         assert g.markdown.convert(dedent('''\
             # Header
@@ -602,7 +601,7 @@ def test_markdown_autolink_with_escape(self):
 
     def test_markdown_invalid_script(self):
         r = g.markdown.convert('<script>alert(document.cookies)</script>')
-        assert '<div class="markdown_content">\n</div>' == r
+        assert '<div class="markdown_content">&lt;script&gt;alert(document.cookies)&lt;/script&gt;\n</div>' == r
 
     def test_markdown_invalid_onerror(self):
         r = g.markdown.convert('<img src=x onerror=alert(document.cookie)>')
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index ae318e998..5ea729e59 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -779,6 +779,6 @@ def test_querystring():
 
 
 def test_clean_html():
-    assert h.clean_html('<script>alert(1)</script>') == ''
+    assert h.clean_html('<script>alert(1)</script>') == '&lt;script&gt;alert(1)&lt;/script&gt;'
     assert h.clean_html('<b style="color: red; right: 0">ok</b>') == '<b style="color: red;">ok</b>'
     assert isinstance(h.clean_html('foo'), Markup)
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index 797f3d04b..d66b5e434 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -248,7 +248,8 @@ def sanitize_html(self, html):
             return utils.ForgeHTMLSanitizer().sanitize(html)
 
     def test_html_sanitizer_iframe(self):
-        assert self.sanitize_html('<div><iframe></iframe></div>') == '<div></div>'
+        assert (self.sanitize_html('<div><iframe></iframe></div>') ==
+                '<div>&lt;iframe&gt;&lt;/iframe&gt;</div>')
 
     def test_html_sanitizer_youtube_iframe(self):
         assert (self.sanitize_html(
@@ -260,17 +261,20 @@ def test_html_sanitizer_youtube_iframe(self):
             '<div><iframe src="https://www.youtube-nocookie.com/embed/kOLpSPEA72U?feature=oembed"></iframe></div>')
 
         assert (self.sanitize_html('<div><iframe src="https://evil.com/embed/x"></iframe></div>') ==
-                '<div></div>')
+                '<div>&lt;iframe src="https://evil.com/embed/x"&gt;&lt;/iframe&gt;</div>')
 
     def test_html_sanitizer_form_elements(self):
         assert (self.sanitize_html('<p>test</p><form method="post" action="http://localhost/foo.php">'
                                    '<input type=file><input type=text><textarea>asdf</textarea></form>') ==
-                '<p>test</p>')
+                '<p>test</p>&lt;form method="post" action="http://localhost/foo.php"&gt;'
+                '&lt;input type="file"&gt;&lt;input type="text"&gt;'
+                '&lt;textarea&gt;asdf&lt;/textarea&gt;&lt;/form&gt;')
 
     def test_html_sanitizer_checkbox(self):
         assert (self.sanitize_html('<p><input type="checkbox" disabled/><input type="text" disabled/>'
                                    '<input type="checkbox" disabled checked/></p>') ==
-                '<p><input type="checkbox" disabled=""><input type="checkbox" disabled="" checked=""></p>')
+                '<p><input type="checkbox" disabled="">&lt;input type="text" disabled=""&gt;'
+                '<input type="checkbox" disabled="" checked=""></p>')
 
     def test_html_sanitizer_summary(self):
         assert (self.sanitize_html('<details open="open"><summary>An Summary</summary><ul><li>Bullet Item</li></ul></details>') ==
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
index a0bdcc9a2..e25581ffd 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -157,9 +157,8 @@ def test_solarize_strips_markdown(self):
         assert self.obj.solarize() == dict(text='Header')
 
     def test_solarize_html_in_text(self):
-        # raw script tag is removed entirely by sanitization, so not indexed (matching what is displayed)
         self.obj.index = lambda: dict(text='<script>a(1)</script>')
-        assert self.obj.solarize() == dict(text='')
+        assert self.obj.solarize() == dict(text='<script>a(1)</script>')
         self.obj.index = lambda: dict(text='&lt;script&gt;a(1)&lt;/script&gt;')
         assert self.obj.solarize() == dict(text='<script>a(1)</script>')