You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2025/08/21 20:49:15 UTC

(allura) branch master updated: [#8582] html sanitization: remove most classes, prefix ids. Also prefix auto-generated section ids

This is an automated email from the ASF dual-hosted git repository.

brondsem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git


The following commit(s) were added to refs/heads/master by this push:
     new 4cb04db5d [#8582] html sanitization: remove most classes, prefix ids.  Also prefix auto-generated section ids
4cb04db5d is described below

commit 4cb04db5dc44ca735e8ac8500c861485d115237d
Author: Guillermo Cruz <gu...@slashdotmedia.com>
AuthorDate: Wed Aug 13 13:27:34 2025 -0600

    [#8582] html sanitization: remove most classes, prefix ids.  Also prefix auto-generated section ids
---
 Allura/allura/lib/app_globals.py                   | 11 +++++-
 Allura/allura/lib/macro.py                         |  2 +-
 Allura/allura/lib/utils.py                         | 44 +++++++++++++++++++++-
 Allura/allura/public/nf/js/allura-base.js          | 12 ++++++
 .../templates/widgets/project_screenshots.html     |  6 +--
 Allura/allura/tests/functional/test_newforge.py    | 11 ++++--
 Allura/allura/tests/test_globals.py                | 28 +++++++++-----
 Allura/allura/tests/test_helpers.py                |  2 +-
 Allura/development.ini                             |  5 +++
 .../forgeimporters/github/tests/test_wiki.py       |  2 +-
 .../forgetracker/tests/functional/test_root.py     |  2 +-
 11 files changed, 102 insertions(+), 23 deletions(-)

diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 0e9c72102..cfc46b8e1 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -33,6 +33,7 @@
 import pygments.lexers
 import pygments.formatters
 import pygments.util
+from markdown.extensions import toc
 from tg import config
 from tg import request
 from tg import tmpl_context as c
@@ -80,6 +81,10 @@ def make_markdown_instance(self, **forge_ext_kwargs):
         # but ForgeExtension and the classes it makes often save things on `self` which might not be thread safe
         # when multiple threads are using the same instance
         # TestCachedMarkdown re-uses the same instance previously, and had a bug that AddCustomClass wasn't running
+
+        def toc_slugify_with_prefix(value, separator):
+            return 'h-' + toc.slugify(value, separator)
+
         return markdown.Markdown(
             extensions=['markdown.extensions.fenced_code', 'markdown.extensions.codehilite',
                         'markdown.extensions.abbr', 'markdown.extensions.def_list', 'markdown.extensions.footnotes',
@@ -87,7 +92,9 @@ def make_markdown_instance(self, **forge_ext_kwargs):
                         ForgeExtension(**forge_ext_kwargs),
                         EmojiExtension(),
                         UserMentionExtension(),
-                        'markdown.extensions.tables', 'markdown.extensions.toc', 'markdown.extensions.nl2br',
+                        'markdown.extensions.tables',
+                        toc.TocExtension(slugify=toc_slugify_with_prefix),
+                        'markdown.extensions.nl2br',
                         'markdown_checklist.extension'],
             output_format='html')
 
@@ -126,7 +133,7 @@ def cached_convert(self, artifact: MappedClass, field_name: str) -> Markup:
                 field_name, artifact.__class__.__name__)
             return self.convert(source_text)
 
-        bugfix_rev = 4  # increment this if we need all caches to invalidated (e.g. xss in markdown rendering fixed)
+        bugfix_rev = 5  # increment this if we need all caches to invalidated (e.g. xss in markdown rendering fixed)
         md5 = None
         # If a cached version exists and it is valid, return it.
         if cache.md5 is not None:
diff --git a/Allura/allura/lib/macro.py b/Allura/allura/lib/macro.py
index e073c35b6..61dbea85a 100644
--- a/Allura/allura/lib/macro.py
+++ b/Allura/allura/lib/macro.py
@@ -288,7 +288,7 @@ def get_projects_for_macro(
             for p in M.Project.query.find(q):
                 if h.has_access(p, 'read'):
                     total = total + 1
-        response = '<p class="macro_projects_total">%s Projects</p>%s' % \
+        response = '<p>%s Projects</p>%s' % \
             (total, response)
     return response
 
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index 0391f026c..b9625114f 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -48,7 +48,7 @@
 from tg import redirect, app_globals as g
 from tg.decorators import before_validate
 from tg.controllers.util import etag_cache
-from paste.deploy.converters import asbool, asint
+from paste.deploy.converters import asbool, asint, aslist
 from markupsafe import Markup
 from webob import exc
 from pygments.formatters import HtmlFormatter
@@ -555,11 +555,36 @@ def __init__(self, *args, **kwargs):
 
         # srcset is used in our own project_list/project_summary widgets
         # which are used as macros so go through markdown
-        self.allowed_attributes = html5lib.filters.sanitizer.allowed_attributes | {(None, 'srcset')}
+        self.allowed_attributes = (html5lib.filters.sanitizer.allowed_attributes | {(None, 'srcset')})
 
         self.valid_iframe_srcs = ('https://www.youtube.com/embed/',
                                   'https://www.youtube-nocookie.com/embed/',
                                   )
+
+        self.valid_class_values = {
+            'markdown_content',  # our wrapper class
+            # standard extensions:
+            'codehilite', 'footnote', 'checklist', 'toc', 'footnote-ref', 'footnote-backref',
+            # macros:
+            'user-mention', 'proj_icon', 'list', 'card', 'feature', 'box', 'notch', 'desc',
+            'neighborhood_feed_entry', 'md-users-list', 'md-users-list-more',
+            # ForgeLinkPattern:
+            'strikethrough', 'alink', 'notfound',
+            # codehilite classes, and our extra related classes:
+            'p', 'n', 'hll', 'c', 'k', 'o', 'cm', 'cp', 'c1', 'cs', 'gd', 'ge', 'gr', 'gh', 'gi', 'go', 'gp', 'gs',
+            'gu', 'gt', 'kc', 'kd', 'kn', 'kp', 'kr', 'kt', 'm', 's', 'na', 'nb', 'nc', 'no', 'nd', 'ni', 'ne', 'nf',
+            'nl', 'nn', 'nt', 'nv', 'ow', 'w', 'mf', 'mh', 'mi', 'mo', 'sb', 'sc', 'sd', 's2', 'se', 'sh', 'si', 'sx',
+            'sr', 's1', 'ss', 'bp', 'vc', 'vg', 'vi', 'il', 'code_block', 'lineno', 'codehilitetable', 'linenodiv',
+            'linenos', 'ch', 'code', 'normal', 'err',
+            # helpful classes:
+            'error', 'notice', 'success', 'ok', 'info',
+            'fa',
+        } | set(aslist(tg.config.get('safe_html.classes', [])))
+        self.valid_partial_class_prefixes = tuple(['fa-'] + aslist(tg.config.get('safe_html.class_prefixes', None)))
+        self.valid_id_prefixes = {
+            'h-', # see toc_slugify_with_prefix
+            'fn:', 'fnref:', # from footnotes extension
+        } | set(aslist(tg.config.get('safe_html.id_prefixes', [])))
         self._prev_token_was_ok_iframe = False
 
     def sanitize_token(self, token):
@@ -584,6 +609,21 @@ def sanitize_token(self, token):
             elif token.get('type') == "EndTag" and self._prev_token_was_ok_iframe:
                 self.allowed_elements.add(iframe_el)
 
+        # sanitize classes and ids
+        if token.get('type') == 'StartTag':
+            classes = token.get('data', {}).get((None, 'class'), '')
+            if classes:
+                classes = classes.split()
+                cleaned_classes = [c for c in classes if c in self.valid_class_values or c.startswith(self.valid_partial_class_prefixes)]
+                if cleaned_classes != classes:
+                    log.info(f'Removed invalid classes: {classes} => {cleaned_classes}')
+                token['data'][(None, 'class')] = ' '.join(cleaned_classes)
+
+            id = token.get('data', {}).get((None, 'id'), '')
+            if id:
+                if not any(id.startswith(prefix) for prefix in self.valid_id_prefixes):
+                    token['data'][(None, 'id')] = 'user-content-' + id
+
         self._prev_token_was_ok_iframe = ok_opening_iframe
 
         if token.get('name') == 'input':
diff --git a/Allura/allura/public/nf/js/allura-base.js b/Allura/allura/public/nf/js/allura-base.js
index 6837960cc..54aabdf56 100644
--- a/Allura/allura/public/nf/js/allura-base.js
+++ b/Allura/allura/public/nf/js/allura-base.js
@@ -239,6 +239,18 @@ $(function(){
 
         e.preventDefault();
     });
+    // backwards compatibility for old anchors.  If there's an intended target, but not an active one, try again with TOC prefix
+    if (window.location.hash && !document.querySelector(':target')) {
+        var new_hash = window.location.hash.replace('#', '#h-');
+        if (document.querySelector(new_hash)) {
+            window.location.hash = new_hash;
+        } else {
+            new_hash = window.location.hash.replace('#', '#user-content-');
+            if (document.querySelector(new_hash)) {
+                window.location.hash = new_hash;
+            }
+        }
+    }
 });
 
 // Interactive checkboxes
diff --git a/Allura/allura/templates/widgets/project_screenshots.html b/Allura/allura/templates/widgets/project_screenshots.html
index 5aa5059d2..278d183b9 100644
--- a/Allura/allura/templates/widgets/project_screenshots.html
+++ b/Allura/allura/templates/widgets/project_screenshots.html
@@ -23,9 +23,9 @@
 {% endif %}
 <div class="{% if draggable %}sortable{% endif %}">
   {% for ss in screenshots %}
-  <div data-ss-id="{{ ss._id }}" class="screenshot">
-    <div class="image">
-      <a class="lightbox" href="{{project.url()}}screenshot/{{h.urlquote(ss.filename)}}" ><img src="{{project.url()}}screenshot/{{h.urlquote(ss.filename)}}/thumb" alt="Screenshot thumbnail"/></a>
+  <div data-ss-id="{{ ss._id }}" {% if edit %}class="screenshot"{% endif %}>
+    <div {% if edit %}class="image"{% endif %}>
+      <a {% if edit %}class="lightbox"{% endif %} href="{{project.url()}}screenshot/{{h.urlquote(ss.filename)}}" ><img src="{{project.url()}}screenshot/{{h.urlquote(ss.filename)}}/thumb" alt="Screenshot thumbnail"/></a>
       {% if not edit %}
       <br>
       {{ss.caption}}
diff --git a/Allura/allura/tests/functional/test_newforge.py b/Allura/allura/tests/functional/test_newforge.py
index 92b4cfe01..69767118c 100644
--- a/Allura/allura/tests/functional/test_newforge.py
+++ b/Allura/allura/tests/functional/test_newforge.py
@@ -14,10 +14,11 @@
 #       KIND, either express or implied.  See the License for the
 #       specific language governing permissions and limitations
 #       under the License.
-
-
+import logging
 from urllib.parse import quote
 
+from testfixtures import LogCapture
+
 from allura.tests import TestController
 from allura.tests import decorators as td
 from allura import model as M
@@ -43,10 +44,14 @@ def test_markdown_to_html(self):
                          status=400)
 
     def test_markdown_syntax(self):
-        r = self.app.get('/nf/markdown_syntax')
+        with LogCapture(level=logging.INFO) as logs:
+            r = self.app.get('/nf/markdown_syntax')
         r.mustcontain('Markdown Syntax')
         r.mustcontain('href="http://someurl"')
 
+        markdown_invalid_classes = [log[2] for log in logs if 'invalid class' in log[2]]
+        assert not markdown_invalid_classes
+
     def test_markdown_syntax_dialog(self):
         r = self.app.get('/nf/markdown_syntax_dialog')
         r.mustcontain('<h1>Markdown Syntax Guide</h1>')
diff --git a/Allura/allura/tests/test_globals.py b/Allura/allura/tests/test_globals.py
index e090496b1..c93abed71 100644
--- a/Allura/allura/tests/test_globals.py
+++ b/Allura/allura/tests/test_globals.py
@@ -188,10 +188,10 @@ def test_macro_projects(self):
             assert 'alt="Test Project Logo"' in r, r
             assert 'alt="A Subproject Logo"' in r, r
             r = g.markdown_wiki.convert('[[projects show_total=True sort=random]]')
-            assert '<p class="macro_projects_total">3 Projects' in r, r
+            assert '<p>3 Projects' in r, r
             r = g.markdown_wiki.convert(
                 '[[projects show_total=True private=True sort=random]]')
-            assert '<p class="macro_projects_total">1 Projects' in r, r
+            assert '<p>1 Projects' in r, r
             assert 'alt="Test 2 Logo"' in r, r
             assert 'alt="Test Project Logo"' not in r, r
             assert 'alt="A Subproject Logo"' not in r, r
@@ -387,8 +387,8 @@ def test_markdown_toc(self):
                 ## Header 2"""))
         assert dedent('''\
             <ul>
-            <li><a href="#header-1">Header 1</a><ul>
-            <li><a href="#header-2">Header 2</a></li>
+            <li><a href="#h-header-1">Header 1</a><ul>
+            <li><a href="#h-header-2">Header 2</a></li>
             </ul>
             </li>
             </ul>''') in r
@@ -447,11 +447,11 @@ def test_markdown_basics(self):
         with h.push_context('test', 'wiki', neighborhood='Projects'):
             text = g.markdown.convert('# Foo!\n[Home]')
             assert (text ==
-                    '<div class="markdown_content"><h1 id="foo">Foo!</h1>\n'
+                    '<div class="markdown_content"><h1 id="h-foo">Foo!</h1>\n'
                     '<p><a class="alink" href="/p/test/wiki/Home/">[Home]</a></p></div>')
             text = g.markdown.convert('# Foo!\n[Rooted]')
             assert (text ==
-                    '<div class="markdown_content"><h1 id="foo">Foo!</h1>\n'
+                    '<div class="markdown_content"><h1 id="h-foo">Foo!</h1>\n'
                     '<p><span>[Rooted]</span></p></div>')
 
         assert (
@@ -476,7 +476,7 @@ def test_markdown_basics(self):
                 for i in range(10):
                     print i
             ''')) == dedent('''\
-                <div class="markdown_content"><h1 id="header">Header</h1>
+                <div class="markdown_content"><h1 id="h-header">Header</h1>
                 <p>Some text in a regular paragraph</p>
                 <div class="codehilite"><pre><span></span><code><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
                     <span class="nb">print</span> <span class="n">i</span>
@@ -590,6 +590,16 @@ def test_markdown_invalid_script_in_link2(self):
                 '''href='http://"&gt;&lt;img%20src=x%20onerror=alert(document.cookie)&gt;' '''
                 'rel="nofollow">xss</a></p></div>' == r)
 
+    def test_markdown_invalid_classes_ids(self):
+        r = g.markdown.convert("# Test message")
+        assert r == '<div class="markdown_content"><h1 id="h-test-message">Test message</h1></div>'
+
+        r = g.markdown.convert('<p class="markdown-class notice" id="markdown-id">This is a test</p>')
+        assert r == '<div class="markdown_content"><p class="notice" id="user-content-markdown-id">This is a test</p>\n</div>'
+
+        r = g.markdown.convert('<i class="fa fa-cog">gear icon</i>')
+        assert r == '<div class="markdown_content"><p><i class="fa fa-cog">gear icon</i></p></div>'
+
     def test_markdown_extremely_slow(self):
         # regex-as-re-globally package mitigates this being slow; also see `NOBRACKET` comments
         g.markdown.convert(inspect.cleandoc('''bonjour, voila ce que j'obtient en voulant ajouter un utilisateur a un groupe de sécurite, que ce soit sur un groupe pre-existant, ou sur un groupe crée.
@@ -606,8 +616,8 @@ def test_markdown_extremely_slow(self):
         assert True   # finished!
 
     def test_macro_include(self):
-        r = g.markdown.convert('[[include ref=Home id=foo]]')
-        assert '<div id="foo">' in r, r
+        r = g.markdown.convert('[[include ref=Home id=foo class=modal data-a=b]]')
+        assert '<div class="" id="user-content-foo">' in r, r
         assert 'href="./foo"' in g.markdown.convert('[My foo](foo)')
         assert 'href="..' not in g.markdown.convert('[My foo](./foo)')
 
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index d9f223479..752477303 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -273,7 +273,7 @@ def test_render_any_markup_plain():
 def test_render_any_markup_formatting():
     assert (str(h.render_any_markup('README.md', '### foo\n'
                                     '    <script>alert(1)</script>bar')) ==
-            '<div class="markdown_content"><h3 id="foo">foo</h3>\n'
+            '<div class="markdown_content"><h3 id="h-foo">foo</h3>\n'
             '<div class="codehilite"><pre><span></span><code><span class="nt">'
             '&lt;script&gt;</span>alert(1)<span class="nt">'
             '&lt;/script&gt;</span>bar\n</code></pre></div>\n</div>')
diff --git a/Allura/development.ini b/Allura/development.ini
index a29b4027b..2405801b8 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -662,6 +662,11 @@ markdown_render_max_length = 200000
 ; Don't add rel=nofollow to these domains when generating links from Markdown content
 ;nofollow_exempt_domains =
 
+; when sanitizing HTML generated from markdown and other sources, allow these classes & ids, in addition to the built-in defaults:
+;safe_html.classes =
+;safe_html.class_prefixes =
+;safe_html.id_prefixes =
+
 ; By default project admins can soft-delete their projects.
 ; A soft-deleted project will still be in the database and visible to admins, but not to others.
 allow_project_delete = true
diff --git a/ForgeImporters/forgeimporters/github/tests/test_wiki.py b/ForgeImporters/forgeimporters/github/tests/test_wiki.py
index d3b53dbda..a137a8e0d 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_wiki.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_wiki.py
@@ -28,7 +28,7 @@
 from forgeimporters.github.wiki import GitHubWikiImporter
 from forgeimporters.github.utils import GitHubMarkdownConverter
 from forgeimporters.github import GitHubOAuthMixin
-
+from alluratest.controller import setup_basic_test
 
 # important to be distinct from 'test' which ForgeWiki uses, so that the
 # tests can run in parallel and not clobber each other
diff --git a/ForgeTracker/forgetracker/tests/functional/test_root.py b/ForgeTracker/forgetracker/tests/functional/test_root.py
index 6661b7ac2..4a3a4d46e 100644
--- a/ForgeTracker/forgetracker/tests/functional/test_root.py
+++ b/ForgeTracker/forgetracker/tests/functional/test_root.py
@@ -803,7 +803,7 @@ def test_cached_convert(self):
 
         # Make sure the cache has been saved.
         t = tm.Ticket.query.find({'_id': ticket._id}).first()
-        assert '<h1 id="test-markdown-cached_convert">Test markdown cached_convert</h1>' in t.description_cache.html
+        assert '<h1 id="h-test-markdown-cached_convert">Test markdown cached_convert</h1>' in t.description_cache.html
 
     def test_ticket_diffs(self):
         self.new_ticket(summary='difftest', description='1\n2\n3\n')