You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2019/03/12 21:45:29 UTC
[allura] branch db/8270 created (now eead860)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8270
in repository https://gitbox.apache.org/repos/asf/allura.git.
at eead860 [#8270] sitemap skips external link tools by default, make tool skips configurable
This branch includes the following new commits:
new 253b4a6 [#8270] rel=nofollow on any menu items going external; include current domain as exempt from nofollow
new eead860 [#8270] sitemap skips external link tools by default, make tool skips configurable
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[allura] 02/02: [#8270] sitemap skips external link tools by
default, make tool skips configurable
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8270
in repository https://gitbox.apache.org/repos/asf/allura.git
commit eead860d3eaae052c7a26ee0048f9cf452772b0e
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Tue Mar 12 17:38:20 2019 -0400
[#8270] sitemap skips external link tools by default, make tool skips configurable
---
Allura/allura/scripts/create_sitemap_files.py | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/Allura/allura/scripts/create_sitemap_files.py b/Allura/allura/scripts/create_sitemap_files.py
index bd34320..5bd3bb6 100644
--- a/Allura/allura/scripts/create_sitemap_files.py
+++ b/Allura/allura/scripts/create_sitemap_files.py
@@ -101,17 +101,17 @@ class CreateSitemapFiles(ScriptTask):
locs = []
file_count = 0
- nbhd_id = []
- if options.neighborhood:
- prefix = ['/%s/' % n for n in options.neighborhood]
- nbhd_id = [nbhd._id for nbhd in M.Neighborhood.query.find({'url_prefix': {'$in': prefix}})]
+ excl_nbhd_ids = []
+ if options.exclude_neighborhoods:
+ prefix = ['/%s/' % n for n in options.exclude_neighborhoods]
+ excl_nbhd_ids = [nbhd._id for nbhd in M.Neighborhood.query.find({'url_prefix': {'$in': prefix}})]
# write sitemap files, MAX_SITEMAP_URLS per file
- for chunk in utils.chunked_find(M.Project, {'deleted': False, 'neighborhood_id': {'$nin': nbhd_id}}):
+ for chunk in utils.chunked_find(M.Project, {'deleted': False, 'neighborhood_id': {'$nin': excl_nbhd_ids}}):
for p in chunk:
c.project = p
try:
- for s in p.sitemap(excluded_tools=['git', 'hg', 'svn']):
+ for s in p.sitemap(excluded_tools=options.exclude_tools):
url = config['base_url'] + s.url if s.url[0] == '/' else s.url
locs.append({'url': url,
'date': p.last_updated.strftime("%Y-%m-%d")})
@@ -161,9 +161,13 @@ class CreateSitemapFiles(ScriptTask):
help='Number of URLs per sitemap file. [default: %(default)s, max: ' +
str(MAX_SITEMAP_URLS) + ']',
action=Validate)
- parser.add_argument('-n', '--neighborhood', dest='neighborhood',
+ parser.add_argument('--exclude-neighborhood', '-n', '--neighborhood', dest='exclude_neighborhoods',
+ help="URL prefix of excluded neighborhood(s) Example: u",
+ default=None, nargs='*', metavar='N')
+ parser.add_argument('--exclude-tools', dest='exclude_tools',
help="URL prefix of excluded neighborhood(s)",
- default=None, nargs='*')
+ default=['link', 'git', 'hg', 'svn'],
+ nargs='*', metavar='TOOL')
parser.add_argument('--url-dir', dest='url_dir',
default='/allura_sitemap',
help='URL directory in which the files will be served from')
@@ -173,5 +177,6 @@ class CreateSitemapFiles(ScriptTask):
def get_parser():
return CreateSitemapFiles.parser()
+
if __name__ == '__main__':
CreateSitemapFiles.main()
[allura] 01/02: [#8270] rel=nofollow on any menu items going
external; include current domain as exempt from nofollow
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8270
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 253b4a6cafc67a3cd02ab0d8a09a6df9d2e46b45
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Tue Mar 12 17:14:51 2019 -0400
[#8270] rel=nofollow on any menu items going external; include current domain as exempt from nofollow
---
.../ext/user_profile/templates/sections/tools.html | 4 ++--
Allura/allura/lib/markdown_extensions.py | 8 +++-----
Allura/allura/lib/utils.py | 9 ++++++++-
Allura/allura/model/project.py | 5 ++++-
Allura/allura/templates/jinja_master/top_nav.html | 4 ++--
Allura/allura/tests/test_utils.py | 16 ++++++++++++++++
6 files changed, 35 insertions(+), 11 deletions(-)
diff --git a/Allura/allura/ext/user_profile/templates/sections/tools.html b/Allura/allura/ext/user_profile/templates/sections/tools.html
index 1a39e91..55aad35 100644
--- a/Allura/allura/ext/user_profile/templates/sections/tools.html
+++ b/Allura/allura/ext/user_profile/templates/sections/tools.html
@@ -28,11 +28,11 @@
<ul>
{% for tool in c.project.grouped_navbar_entries() %}
<li>
- <a href="{{tool.url}}">{{ tool.label }}</a>
+ <a href="{{tool.url}}" {{ tool.extra_html_attrs|xmlattr }}>{{ tool.label }}</a>
{% if tool.children %}
<ul>
{% for child in tool.children %}
- <li><a href="{{child.url}}">{{ child.label }}</a></li>
+ <li><a href="{{child.url}}" {{ child.extra_html_attrs|xmlattr }}>{{ child.label }}</a></li>
{% endfor %}
</ul>
{% endif %}
diff --git a/Allura/allura/lib/markdown_extensions.py b/Allura/allura/lib/markdown_extensions.py
index d025590..09ff02a 100644
--- a/Allura/allura/lib/markdown_extensions.py
+++ b/Allura/allura/lib/markdown_extensions.py
@@ -32,7 +32,7 @@ import emoji
from . import macro
from . import helpers as h
from allura import model as M
-from allura.lib.utils import ForgeHTMLSanitizerFilter
+from allura.lib.utils import ForgeHTMLSanitizerFilter, is_nofollow_url
log = logging.getLogger(__name__)
@@ -473,10 +473,8 @@ class RelativeLinkRewriter(markdown.postprocessors.Postprocessor):
val = val.replace(' ', '%20')
tag[attr] = val
if '://' in val:
- for domain in re.split(r'\s*,\s*', config.get('nofollow_exempt_domains', '')):
- if domain and domain in val:
- return
- tag['rel'] = 'nofollow'
+ if is_nofollow_url(val):
+ tag['rel'] = 'nofollow'
return
if val.startswith('/'):
return
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index e07323c..7198386 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -30,6 +30,7 @@ import magic
from itertools import groupby
import operator as op
import collections
+from urlparse import urlparse
import tg
import emoji
@@ -792,4 +793,10 @@ def get_key_from_value(d, val):
for k, v in d.items():
if val in v:
return k
- return ''
\ No newline at end of file
+ return ''
+
+
+def is_nofollow_url(url):
+ url_domain = urlparse(url).hostname
+ ok_domains = re.split(r'\s*,\s*', tg.config.get('nofollow_exempt_domains', '')) + [tg.config['domain']]
+ return url_domain and url_domain not in ok_domains
diff --git a/Allura/allura/model/project.py b/Allura/allura/model/project.py
index 1a70914..8fa6167 100644
--- a/Allura/allura/model/project.py
+++ b/Allura/allura/model/project.py
@@ -33,7 +33,6 @@ import formencode as fe
from webob import exc
import PIL
-from allura.lib.decorators import memoize
from ming import schema as S
from ming.utils import LazyProperty
from ming.orm import ThreadLocalORMSession
@@ -46,8 +45,10 @@ from allura.lib import plugin
from allura.lib import exceptions
from allura.lib import security
from allura.lib import validators as v
+from allura.lib.decorators import memoize
from allura.lib.security import has_access
from allura.lib.search import SearchIndexable
+from allura.lib.utils import is_nofollow_url
from allura.model.types import MarkdownCache
from .session import main_orm_session
@@ -626,6 +627,8 @@ class Project(SearchIndexable, MappedClass, ActivityNode, ActivityObject):
entry = sm.bind_app(app)
entry.tool_name = ac.tool_name
entry.ui_icon = 'tool-%s' % entry.tool_name.lower()
+ if is_nofollow_url(entry.url):
+ entry.extra_html_attrs = {'rel': 'nofollow'}
if not self.is_nbhd_project and (entry.tool_name.lower() in anchored_tools.keys()):
ordinal = anchored_tools.keys().index(
entry.tool_name.lower())
diff --git a/Allura/allura/templates/jinja_master/top_nav.html b/Allura/allura/templates/jinja_master/top_nav.html
index ae8fefe..ac3727d 100644
--- a/Allura/allura/templates/jinja_master/top_nav.html
+++ b/Allura/allura/templates/jinja_master/top_nav.html
@@ -21,14 +21,14 @@
<ul class="dropdown">
{% for s in c.project.grouped_navbar_entries() %}
<li class="{% if s.matches_url(request) %}selected{% endif %}">
- <a href="{{s.url}}" class="tool-{{(s.tool_name or 'admin').lower()}}-32">
+ <a href="{{s.url}}" class="tool-{{(s.tool_name or 'admin').lower()}}-32" {{ s.extra_html_attrs|xmlattr }}>
{{s.label}}
</a>
{% set grouped_tool_count = s.matching_urls|length %}
{% if grouped_tool_count %}
<ul>
{%for tool in s.children%}
- <li class="{% if tool.matches_url(request) %}selected{% endif %}"><a href="{{tool.url}}">{{tool.label}}</a></li>
+ <li class="{% if tool.matches_url(request) %}selected{% endif %}"><a href="{{tool.url}}" {{ tool.extra_html_attrs|xmlattr }}>{{tool.label}}</a></li>
{%endfor%}
</ul>
{% endif %}
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index 0afc937..5f642a7 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -382,3 +382,19 @@ def unique_attachments():
attachments = [pic1, file1, pic2, file2, pic2, other]
expected = [file2, other, pic2]
assert_equal(expected, ua(attachments))
+
+
+def test_is_nofollow_url():
+ with patch.dict(config, {'domain': 'localhost'}):
+ assert not utils.is_nofollow_url('relative/path')
+ assert not utils.is_nofollow_url('http://localhost/path')
+ assert utils.is_nofollow_url('http://google.com/')
+ assert utils.is_nofollow_url('https://google.com/')
+
+ with patch.dict(config, {'domain': 'localhost',
+ 'nofollow_exempt_domains': 'foo.com, bar.io'}):
+ assert utils.is_nofollow_url('http://google.com/')
+ assert utils.is_nofollow_url('http://xfoo.com/')
+ assert not utils.is_nofollow_url('http://foo.com/')
+ assert not utils.is_nofollow_url('http://bar.io/')
+ assert utils.is_nofollow_url('http://bar.iot/')