You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2026/03/31 16:19:46 UTC
(allura) 01/01: [#8599] detect potentially misleading links and show the destination domain afterwards
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8599
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 7ccfb132eebbe2d3962085e4364a48666ed6b98b
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Thu Mar 26 13:37:38 2026 -0400
[#8599] detect potentially misleading links and show the destination domain afterwards
---
Allura/allura/lib/app_globals.py | 2 +-
Allura/allura/lib/utils.py | 164 +++++++++++++++++++++++++++++++++++++-
Allura/allura/tests/test_utils.py | 105 +++++++++++++++++++++++-
requirements.in | 1 +
4 files changed, 269 insertions(+), 3 deletions(-)
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 72c798777..4b3381014 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -135,7 +135,7 @@ def cached_convert(self, artifact: MappedClass, field_name: str) -> Markup:
field_name, artifact.__class__.__name__)
return self.convert(source_text)
- bugfix_rev = 6 # increment this if we need all caches to invalidated (e.g. xss in markdown rendering fixed)
+ bugfix_rev = 7 # increment this if we need all caches to invalidated (e.g. xss in markdown rendering fixed)
md5 = None
# If a cached version exists and it is valid, return it.
if cache.md5 is not None:
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index c2f3778b6..e680bf2f7 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -29,6 +29,7 @@
import random
import mimetypes
import re
+import unicodedata
from pathlib import Path
from typing import TypeVar
import magic
@@ -47,6 +48,7 @@
from collections import OrderedDict
from bs4 import BeautifulSoup
+import idna
from tg import redirect, app_globals as g
from tg.decorators import before_validate
from tg.controllers.util import etag_cache
@@ -588,6 +590,16 @@ def __init__(self, *args, **kwargs):
'fn:', 'fnref:', # from footnotes extension
} | set(aslist(tg.config.get('safe_html.id_prefixes', [])))
self._prev_token_was_ok_iframe = False
+ self._current_link = None
+ self._pending_link_suffix = None
+
+ def __iter__(self):
+ # override to inject link suffixes in the output
+ for token in super().__iter__():
+ yield token
+ if self._pending_link_suffix is not None:
+ yield self._pending_link_suffix
+ self._pending_link_suffix = None
def sanitize_token(self, token):
"""
@@ -633,7 +645,157 @@ def sanitize_token(self, token):
if attrs.get((None, 'type'), '') == "checkbox":
self.allowed_elements.add(input_el)
- return super().sanitize_token(token)
+ sanitized = super().sanitize_token(token)
+ self._track_link_target(sanitized)
+ return sanitized
+
+ def _track_link_target(self, token):
+ if token is None:
+ return
+
+ tag_type = token.get('type')
+ tag_name = token.get('name')
+
+ if tag_type == 'StartTag' and tag_name == 'a':
+ href = token.get('data', {}).get((None, 'href'))
+ hostname = hostname_from_url(href)
+ self._current_link = {
+ 'href': href,
+ 'hostname': hostname,
+ 'text_parts': [],
+ }
+ return
+
+ if self._current_link is None:
+ return
+
+ if tag_type in ('Characters', 'SpaceCharacters'):
+ self._current_link['text_parts'].append(token.get('data', ''))
+ return
+
+ if tag_type == 'EndTag' and tag_name == 'a':
+ self._finalize_link_target()
+ self._current_link = None
+
+ def _finalize_link_target(self):
+ href = self._current_link['href']
+ hostname = self._current_link['hostname']
+ if not href or not hostname:
+ return
+
+ visible_text = ''.join(self._current_link['text_parts']).strip()
+ if not visible_text:
+ return
+
+ idn_lookalike = 'xn--' in hostname and idn_uses_lookalike_chars(hostname)
+ if not idn_lookalike:
+ # IDN with ASCII-lookalike chars (e.g. Cyrillic/Greek homographs) always
+ # gets the suffix so the deception is visible; all other links only get it
+ # when the visible text looks like a URL pointing somewhere different.
+ if not text_contains_any_url(visible_text):
+ return
+ if text_contains_hostname(visible_text, hostname):
+ return
+
+ self._pending_link_suffix = {
+ 'type': 'Characters',
+ 'data': f' ({hostname})',
+ }
+
+
+# Scripts whose characters are visually distinct from ASCII and unlikely to be used
+# in homograph attacks. Any non-ASCII character whose Unicode name does NOT start
+# with one of these prefixes is treated as a potential ASCII lookalike.
+NON_MISLEADING_IDN_SCRIPTS = frozenset([
+ 'CJK', 'HANGUL', 'HIRAGANA', 'KATAKANA', # East Asian
+ 'ARABIC', 'HEBREW', # Semitic
+ 'THAI', 'LAO', 'KHMER', 'MYANMAR', 'TIBETAN', # Southeast/Central Asian
+ 'DEVANAGARI', 'BENGALI', 'GUJARATI', 'GURMUKHI', 'ORIYA', # South Asian (Indic)
+ 'KANNADA', 'MALAYALAM', 'SINHALA', 'TAMIL', 'TELUGU',
+ 'MONGOLIAN', 'ETHIOPIC', 'GEORGIAN',
+])
+
+
+def idn_uses_lookalike_chars(ascii_hostname: str) -> bool:
+ """
+ Return True if any non-ASCII character in the IDN hostname could be visually
+ confused with an ASCII character (e.g. Cyrillic or Greek lookalikes).
+ """
+ try:
+ unicode_hostname = idna.decode(ascii_hostname)
+ except (idna.IDNAError, UnicodeError):
+ return True # can't decode → assume misleading
+ for char in unicode_hostname:
+ if ord(char) <= 127:
+ continue
+ name = unicodedata.name(char, '')
+ script = name.split()[0] if name else ''
+ if script not in NON_MISLEADING_IDN_SCRIPTS:
+ return True
+ return False
+
+
+def normalize_hostname(hostname: str) -> str:
+ # handles IDN -> ascii punycode
+ if not hostname:
+ return ''
+
+ hostname = hostname.strip().rstrip('.').lower()
+ if not hostname:
+ return ''
+
+ try:
+ return idna.encode(hostname, uts46=True).decode('ascii').lower()
+ except idna.IDNAError:
+ return hostname
+
+
+def hostname_from_url(url: str) -> str:
+ if not url:
+ return ''
+ # Browsers treat ///foo.com as //foo.com (protocol-relative); urlparse does not,
+ # so collapse three or more leading slashes to two before parsing.
+ url = re.sub(r'^/{3,}', '//', url)
+ return normalize_hostname(urlparse(url).hostname)
+
+
+_HOST_TOKEN_RE = re.compile(r'[^\s<>()]+') # a run of non-whitespace, non <>() characters
+
+
+def text_contains_any_url(text: str) -> bool:
+ """Return True if the visible text appears to contain a URL or domain name."""
+ for token in _HOST_TOKEN_RE.findall(text):
+ candidate = token.strip('\'"[]{}()<>,;!?')
+ if not candidate or ('@' in candidate and '://' not in candidate):
+ continue
+ if '://' in candidate:
+ if urlparse(candidate).hostname:
+ return True
+ elif '.' in candidate and re.search(r'[a-zA-Z]', candidate):
+ if urlparse('//' + candidate).hostname:
+ return True
+ return False
+
+
+def text_contains_hostname(text: str, hostname: str) -> bool:
+ normalized_text = text.casefold()
+ if hostname.casefold() in normalized_text:
+ return True
+
+ for token in _HOST_TOKEN_RE.findall(text):
+ candidate = token.strip('\'"[]{}()<>,;!?')
+ if not candidate or ('@' in candidate and '://' not in candidate):
+ continue
+
+ if '://' in candidate:
+ candidate_hostname = urlparse(candidate).hostname
+ else:
+ candidate_hostname = urlparse('//' + candidate).hostname
+
+ if normalize_hostname(candidate_hostname) == hostname:
+ return True
+
+ return False
def ip_address(request):
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index d7174a444..e743b7493 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -32,6 +32,7 @@
from tg import config
import html5lib
import html5lib.treewalkers
+import html5lib.serializer
from alluratest.controller import setup_unit_test
@@ -242,12 +243,20 @@ def greetings(self):
class TestHTMLSanitizer:
+ def setup_method(self, method):
+ setup_unit_test()
+
def walker_from_text(self, text):
parsed = html5lib.parseFragment(text)
TreeWalker = html5lib.treewalkers.getTreeWalker("etree")
walker = TreeWalker(parsed)
return walker
+ def sanitize_html(self, html):
+ with h.push_config(config, domain='mysite.com'):
+ filt = utils.ForgeHTMLSanitizerFilter(self.walker_from_text(html))
+ return html5lib.serializer.HTMLSerializer().render(filt)
+
def simple_tag_list(self, sanitizer):
# no attrs, no close tag flag check, just real simple
return [
@@ -285,6 +294,100 @@ def test_html_sanitizer_summary(self):
p = utils.ForgeHTMLSanitizerFilter(walker)
assert self.simple_tag_list(p) == ['details', 'summary', 'summary', 'ul', 'li', 'li', 'ul', 'details']
+ def test_misleading_links(self):
+ # OK: generic text
+ assert (self.sanitize_html('<a href="http://evil.com/">click here</a>') ==
+ '<a href="http://evil.com/">click here</a>')
+
+ # OK: domain-like text matching actual link
+ assert (self.sanitize_html('<a href="http://evil.com/path">evil.com</a>') ==
+ '<a href="http://evil.com/path">evil.com</a>')
+
+ # SUFFIX: domain-like text that is a different domain
+ assert (self.sanitize_html('<a href="http://evil.com/">example.com</a>') ==
+ '<a href="http://evil.com/">example.com</a> (evil.com)')
+
+ # SUFFIX: domain with protocol, that is a different URL
+ assert (self.sanitize_html('<a href="http://evil.com/">http://example.com/</a>') ==
+ '<a href="http://evil.com/">http://example.com/</a> (evil.com)')
+
+ # SUFFIX: different protocol variations
+ assert (self.sanitize_html('<a href="//evil.com/">http://example.com/</a>') ==
+ '<a href="//evil.com/">http://example.com/</a> (evil.com)')
+ assert (self.sanitize_html('<a href="///evil.com/">http://example.com/</a>') ==
+ '<a href="///evil.com/">http://example.com/</a> (evil.com)')
+
+ # OK: URL text matching actual link
+ assert (self.sanitize_html('<a href="http://evil.com/">http://evil.com/</a>') ==
+ '<a href="http://evil.com/">http://evil.com/</a>')
+
+ # false-positive SUFFIX due to domain name detection .txt looks like a TLD :(
+ assert (self.sanitize_html('<a href="https://example.com/repo/README.txt">README.txt</a>') ==
+ '<a href="https://example.com/repo/README.txt">README.txt</a> (example.com)')
+
+ # OK: internal/relative links
+ assert (self.sanitize_html('<a href="/p/local/wiki/">example.com</a>') ==
+ '<a href="/p/local/wiki/">example.com</a>')
+
+ # SUFFIX: mismatch domain embedded in a longer sentence
+ assert (self.sanitize_html('<a href="http://evil.com/">visit example.com for more</a>') ==
+ '<a href="http://evil.com/">visit example.com for more</a> (evil.com)')
+
+ # SUFFIX: mismatch domain with path in longer sentence
+ assert (self.sanitize_html('<a href="http://evil.com/">see example.com/some/page for details</a>') ==
+ '<a href="http://evil.com/">see example.com/some/page for details</a> (evil.com)')
+
+ # SUFFIX: mismatch full URL in longer sentence
+ assert (self.sanitize_html('<a href="http://evil.com/">see http://example.com/some/page for details</a>') ==
+ '<a href="http://evil.com/">see http://example.com/some/page for details</a> (evil.com)')
+
+ def test_misleading_links_idn(self):
+ # аpple.com with Cyrillic а (U+0430) — homograph of apple.com
+ cyrillic_a = '\u0430'
+
+ # text and href use same IDN domain: always show punycode suffix for IDN lookalike destinations
+ assert (self.sanitize_html(f'<a href="http://{cyrillic_a}pple.com/">{cyrillic_a}pple.com</a>') ==
+ f'<a href="http://{cyrillic_a}pple.com/">{cyrillic_a}pple.com</a> (xn--pple-43d.com)')
+
+ # lookalike unicode text linking to the real domain: suffix reveals real hostname
+ assert (self.sanitize_html(f'<a href="http://apple.com/">{cyrillic_a}pple.com</a>') ==
+ f'<a href="http://apple.com/">{cyrillic_a}pple.com</a> (apple.com)')
+
+ # real-looking text linking to IDN domain: suffix reveals punycode hostname
+ assert (self.sanitize_html(f'<a href="http://{cyrillic_a}pple.com/">apple.com</a>') ==
+ f'<a href="http://{cyrillic_a}pple.com/">apple.com</a> (xn--pple-43d.com)')
+
+ # even generic link text gets the suffix when destination is IDN lookalike
+ assert (self.sanitize_html(f'<a href="http://{cyrillic_a}pple.com/">click here</a>') ==
+ f'<a href="http://{cyrillic_a}pple.com/">click here</a> (xn--pple-43d.com)')
+
+ # Chinese IDN — not an ASCII lookalike, so treated like a normal domain
+ chinese_domain = '中文.com'
+ # generic text: no suffix (Chinese chars are not ASCII lookalikes)
+ assert (self.sanitize_html(f'<a href="http://{chinese_domain}/">click here</a>') ==
+ '<a href="http://中文.com/">click here</a>')
+ # misleading domain text: suffix shown (same as non-IDN behavior)
+ assert (self.sanitize_html(f'<a href="http://{chinese_domain}/">example.com</a>') ==
+ '<a href="http://中文.com/">example.com</a> (xn--fiq228c.com)')
+ # correct domain text: no suffix
+ assert (self.sanitize_html(f'<a href="http://{chinese_domain}/">{chinese_domain}</a>') ==
+ '<a href="http://中文.com/">中文.com</a>')
+
+
+def test_text_contains_any_url():
+ assert utils.text_contains_any_url('example.com') is True
+ assert utils.text_contains_any_url('foo a1-2-3.space bar') is True
+ assert utils.text_contains_any_url('click here') is False
+ # false positives: file extensions and library names with dots parse as hostnames
+ assert utils.text_contains_any_url('README.md') is True # .md suffix
+ assert utils.text_contains_any_url('Node.js') is True # .js suffix
+
+
+def test_text_contains_hostname():
+ assert utils.text_contains_hostname('visit example.com for info', 'example.com') is True
+ assert utils.text_contains_hostname('visit example.com for info', 'example.comevil.org') is False
+ assert utils.text_contains_hostname('click here', 'example.com') is False
+
def test_ip_address():
req = Mock()
@@ -434,4 +537,4 @@ def test_join_paths_no_traversal(base, paths, expected: str | type[Exception]):
def test_hide_email():
assert utils.hide_email('foo@bar.com') == '<fo...@ba...>'
assert utils.hide_email('email@example.com') == '<em...@ex...>'
- assert utils.hide_email('email@example') == '<em...@ex...>'
\ No newline at end of file
+ assert utils.hide_email('email@example') == '<em...@ex...>'
diff --git a/requirements.in b/requirements.in
index 59ea40dbf..43af7b8bf 100644
--- a/requirements.in
+++ b/requirements.in
@@ -15,6 +15,7 @@ feedparser
FormEncode
GitPython
html5lib
+idna
Jinja2
# Webob uses legacy-cgi and so our tests need it too for cgi.FieldStorage, maybe webob 2 will remove it? https://github.com/Pylons/webob/pull/466
legacy-cgi ; python_full_version >= '3.13'