You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by he...@apache.org on 2015/05/29 22:40:48 UTC
[26/45] allura git commit: [#7878] Used 2to3 to see what issues would
come up
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/lib/markdown_extensions.py
----------------------------------------------------------------------
diff --git a/lib/markdown_extensions.py b/lib/markdown_extensions.py
new file mode 100644
index 0000000..db5533d
--- /dev/null
+++ b/lib/markdown_extensions.py
@@ -0,0 +1,569 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+import logging
+from urllib.parse import urljoin
+
+from tg import config
+from BeautifulSoup import BeautifulSoup
+import html5lib
+import html5lib.serializer
+import html5lib.filters.alphabeticalattributes
+import markdown
+
+from . import macro
+from . import helpers as h
+from allura import model as M
+from allura.lib.utils import ForgeHTMLSanitizer
+
+log = logging.getLogger(__name__)
+
+PLAINTEXT_BLOCK_RE = re.compile(
+ r'(?P<bplain>\[plain\])(?P<code>.*?)(?P<eplain>\[\/plain\])',
+ re.MULTILINE | re.DOTALL
+)
+
+MACRO_PATTERN = r'\[\[([^\]\[]+)\]\]'
+
+
+class CommitMessageExtension(markdown.Extension):
+
+ """Markdown extension for processing commit messages.
+
+ People don't expect their commit messages to be parsed as Markdown. This
+ extension is therefore intentionally minimal in what it does. It knows how
+ to handle Trac-style short refs, will replace short refs with links, and
+ will create paragraphs around double-line breaks. That is *all* it does.
+
+ To make it do more, re-add some inlinePatterns and/or blockprocessors.
+
+ Some examples of the Trac-style refs this extension can parse::
+
+ #100
+ r123
+ ticket:100
+ comment:13:ticket:100
+ source:path/to/file.c@123#L456 (rev 123, lineno 456)
+
+ Trac-style refs will be converted to links to the appropriate artifact by
+ the :class:`PatternReplacingProcessor` preprocessor.
+
+ """
+
+ def __init__(self, app):
+ markdown.Extension.__init__(self)
+ self.app = app
+ self._use_wiki = False
+
+ def extendMarkdown(self, md, md_globals):
+ md.registerExtension(self)
+ # remove default preprocessors and add our own
+ md.preprocessors.clear()
+ md.preprocessors['trac_refs'] = PatternReplacingProcessor(
+ TracRef1(), TracRef2(), TracRef3(self.app))
+ # remove all inlinepattern processors except short refs and links
+ md.inlinePatterns.clear()
+ md.inlinePatterns["link"] = markdown.inlinepatterns.LinkPattern(
+ markdown.inlinepatterns.LINK_RE, md)
+ md.inlinePatterns['short_reference'] = ForgeLinkPattern(
+ markdown.inlinepatterns.SHORT_REF_RE, md, ext=self)
+ # remove all default block processors except for paragraph
+ md.parser.blockprocessors.clear()
+ md.parser.blockprocessors['paragraph'] = \
+ markdown.blockprocessors.ParagraphProcessor(md.parser)
+ # wrap artifact link text in square brackets
+ self.forge_link_tree_processor = ForgeLinkTreeProcessor(md)
+ md.treeprocessors['links'] = self.forge_link_tree_processor
+ # Sanitize HTML
+ md.postprocessors['sanitize_html'] = HTMLSanitizer()
+ # Put a class around markdown content for custom css
+ md.postprocessors['add_custom_class'] = AddCustomClass()
+ md.postprocessors['mark_safe'] = MarkAsSafe()
+
+ def reset(self):
+ self.forge_link_tree_processor.reset()
+
+
+class Pattern(object):
+
+ """Base class for regex patterns used by the :class:`PatternReplacingProcessor`.
+
+ Subclasses must define :attr:`pattern` (a compiled regex), and
+ :meth:`repl`.
+
+ """
+ BEGIN, END = r'(^|\b|\s)', r'($|\b|\s)'
+
+ def sub(self, line):
+ return self.pattern.sub(self.repl, line)
+
+ def repl(self, match):
+ """Return a string to replace ``match`` in the source string (the
+ string in which the match was found).
+
+ """
+ return match.group()
+
+
+class TracRef1(Pattern):
+
+ """Replaces Trac-style short refs with links. Example patterns::
+
+ #100 (ticket 100)
+ r123 (revision 123)
+
+ """
+ pattern = re.compile(r'(?<!\[|\w)([#r]\d+)(?!\]|\w)')
+
+ def repl(self, match):
+ shortlink = M.Shortlink.lookup(match.group(1))
+ if shortlink and not getattr(shortlink.ref.artifact, 'deleted', False):
+ return '[{ref}]({url})'.format(
+ ref=match.group(1),
+ url=shortlink.url)
+ return match.group()
+
+
+class TracRef2(Pattern):
+
+ """Replaces Trac-style short refs with links. Example patterns::
+
+ ticket:100
+ comment:13:ticket:400
+
+ """
+ pattern = re.compile(
+ Pattern.BEGIN + r'((comment:(\d+):)?(ticket:)(\d+))' + Pattern.END)
+
+ def repl(self, match):
+ shortlink = M.Shortlink.lookup('#' + match.group(6))
+ if shortlink and not getattr(shortlink.ref.artifact, 'deleted', False):
+ url = shortlink.url
+ if match.group(4):
+ slug = self.get_comment_slug(
+ shortlink.ref.artifact, match.group(4))
+ slug = '#' + slug if slug else ''
+ url = url + slug
+
+ return '{front}[{ref}]({url}){back}'.format(
+ front=match.group(1),
+ ref=match.group(2),
+ url=url,
+ back=match.group(7))
+ return match.group()
+
+ def get_comment_slug(self, ticket, comment_num):
+ """Given the id of an imported Trac comment, return it's Allura slug.
+
+ """
+ if not ticket:
+ return None
+
+ comment_num = int(comment_num)
+ comments = ticket.discussion_thread.post_class().query.find(dict(
+ discussion_id=ticket.discussion_thread.discussion_id,
+ thread_id=ticket.discussion_thread._id,
+ status={'$in': ['ok', 'pending']},
+ deleted=False)).sort('timestamp')
+
+ if comment_num <= comments.count():
+ return comments.all()[comment_num - 1].slug
+
+
+class TracRef3(Pattern):
+
+ """Replaces Trac-style short refs with links. Example patterns::
+
+ source:trunk/server/file.c@123#L456 (rev 123, lineno 456)
+
+ Creates a link to a specific line of a source file at a specific revision.
+
+ """
+ pattern = re.compile(
+ Pattern.BEGIN + r'((source:)([^@#\s]+)(@(\w+))?(#L(\d+))?)' + Pattern.END)
+
+ def __init__(self, app):
+ super(Pattern, self).__init__()
+ self.app = app
+
+ def repl(self, match):
+ if not self.app:
+ return match.group()
+ file, rev, lineno = (
+ match.group(4),
+ match.group(6) or 'HEAD',
+ '#l' + match.group(8) if match.group(8) else '')
+ url = '{app_url}{rev}/tree/{file}{lineno}'.format(
+ app_url=self.app.url,
+ rev=rev,
+ file=file,
+ lineno=lineno)
+ return '{front}[{ref}]({url}){back}'.format(
+ front=match.group(1),
+ ref=match.group(2),
+ url=url,
+ back=match.group(9))
+
+
+class PatternReplacingProcessor(markdown.preprocessors.Preprocessor):
+
+ """A Markdown preprocessor that searches the source lines for patterns and
+ replaces matches with alternate text.
+
+ """
+
+ def __init__(self, *patterns):
+ self.patterns = patterns or []
+
+ def run(self, lines):
+ new_lines = []
+ for line in lines:
+ for pattern in self.patterns:
+ line = pattern.sub(line)
+ new_lines.append(line)
+ return new_lines
+
+
+class ForgeExtension(markdown.Extension):
+
+ def __init__(self, wiki=False, email=False, macro_context=None):
+ markdown.Extension.__init__(self)
+ self._use_wiki = wiki
+ self._is_email = email
+ self._macro_context = macro_context
+
+ def extendMarkdown(self, md, md_globals):
+ md.registerExtension(self)
+ # allow markdown within e.g. <div markdown>...</div> More info at:
+ # https://github.com/waylan/Python-Markdown/issues/52
+ md.preprocessors['html_block'].markdown_in_raw = True
+ md.preprocessors['fenced-code'] = FencedCodeProcessor()
+ md.preprocessors.add('plain_text_block',
+ PlainTextPreprocessor(md), "_begin")
+ md.preprocessors.add(
+ 'macro_include', ForgeMacroIncludePreprocessor(md), '_end')
+ # this has to be before the 'escape' processor, otherwise weird
+ # placeholders are inserted for escaped chars within urls, and then the
+ # autolink can't match the whole url
+ md.inlinePatterns.add('autolink_without_brackets', AutolinkPattern(
+ r'(http(?:s?)://[a-zA-Z0-9./\-\\_%?&=+#;~:!]+)', md), '<escape')
+ # replace the link pattern with our extended version
+ md.inlinePatterns['link'] = ForgeLinkPattern(
+ markdown.inlinepatterns.LINK_RE, md, ext=self)
+ md.inlinePatterns['short_reference'] = ForgeLinkPattern(
+ markdown.inlinepatterns.SHORT_REF_RE, md, ext=self)
+ # macro must be processed before links
+ md.inlinePatterns.add(
+ 'macro', ForgeMacroPattern(MACRO_PATTERN, md, ext=self), '<link')
+ self.forge_link_tree_processor = ForgeLinkTreeProcessor(md)
+ md.treeprocessors['links'] = self.forge_link_tree_processor
+ # Sanitize HTML
+ md.postprocessors['sanitize_html'] = HTMLSanitizer()
+ # Rewrite all relative links that don't start with . to have a '../'
+ # prefix
+ md.postprocessors['rewrite_relative_links'] = RelativeLinkRewriter(
+ make_absolute=self._is_email)
+ # Put a class around markdown content for custom css
+ md.postprocessors['add_custom_class'] = AddCustomClass()
+ md.postprocessors['mark_safe'] = MarkAsSafe()
+
+ def reset(self):
+ self.forge_link_tree_processor.reset()
+
+
+class ForgeLinkPattern(markdown.inlinepatterns.LinkPattern):
+
+ artifact_re = re.compile(r'((.*?):)?((.*?):)?(.+)')
+
+ def __init__(self, *args, **kwargs):
+ self.ext = kwargs.pop('ext')
+ markdown.inlinepatterns.LinkPattern.__init__(self, *args, **kwargs)
+
+ def handleMatch(self, m):
+ el = markdown.util.etree.Element('a')
+ el.text = m.group(2)
+ is_link_with_brackets = False
+ try:
+ href = m.group(9)
+ except IndexError:
+ href = m.group(2)
+ is_link_with_brackets = True
+ try:
+ title = m.group(13)
+ except IndexError:
+ title = None
+
+ classes = ''
+ if href:
+ if href == 'TOC':
+ return '[TOC]' # skip TOC
+ if self.artifact_re.match(href):
+ href, classes = self._expand_alink(href, is_link_with_brackets)
+ el.set('href', self.sanitize_url(self.unescape(href.strip())))
+ el.set('class', classes)
+ else:
+ el.set('href', '')
+
+ if title:
+ title = markdown.inlinepatterns.dequote(self.unescape(title))
+ el.set('title', title)
+
+ if 'notfound' in classes and not self.ext._use_wiki:
+ text = el.text
+ el = markdown.util.etree.Element('span')
+ el.text = '[%s]' % text
+ return el
+
+ def _expand_alink(self, link, is_link_with_brackets):
+ '''Return (href, classes) for an artifact link'''
+ classes = ''
+ if is_link_with_brackets:
+ classes = 'alink'
+ href = link
+ shortlink = M.Shortlink.lookup(link)
+ if shortlink and shortlink.ref and not getattr(shortlink.ref.artifact, 'deleted', False):
+ href = shortlink.url
+ if getattr(shortlink.ref.artifact, 'is_closed', False):
+ classes += ' strikethrough'
+ self.ext.forge_link_tree_processor.alinks.append(shortlink)
+ elif is_link_with_brackets:
+ href = h.urlquote(link)
+ classes += ' notfound'
+ attach_link = link.split('/attachment/')
+ if len(attach_link) == 2 and self.ext._use_wiki:
+ shortlink = M.Shortlink.lookup(attach_link[0])
+ if shortlink:
+ attach_status = ' notfound'
+ for attach in shortlink.ref.artifact.attachments:
+ if attach.filename == attach_link[1]:
+ attach_status = ''
+ classes += attach_status
+ return href, classes
+
+
+class PlainTextPreprocessor(markdown.preprocessors.Preprocessor):
+
+ '''
+ This was used earlier for [plain] tags that the Blog tool's rss importer
+ created, before html2text did good escaping of all special markdown chars.
+ Can be deprecated.
+ '''
+
+ def run(self, lines):
+ text = "\n".join(lines)
+ while 1:
+ res = PLAINTEXT_BLOCK_RE.finditer(text)
+ for m in res:
+ code = self._escape(m.group('code'))
+ placeholder = self.markdown.htmlStash.store(code, safe=True)
+ text = '%s%s%s' % (
+ text[:m.start()], placeholder, text[m.end():])
+ break
+ else:
+ break
+ return text.split("\n")
+
+ def _escape(self, txt):
+ """ basic html escaping """
+ txt = txt.replace('&', '&')
+ txt = txt.replace('<', '<')
+ txt = txt.replace('>', '>')
+ txt = txt.replace('"', '"')
+ return txt
+
+
+class FencedCodeProcessor(markdown.preprocessors.Preprocessor):
+ pattern = '~~~~'
+
+ def run(self, lines):
+ in_block = False
+ new_lines = []
+ for line in lines:
+ if line.lstrip().startswith(self.pattern):
+ in_block = not in_block
+ continue
+ if in_block:
+ new_lines.append(' ' + line)
+ else:
+ new_lines.append(line)
+ return new_lines
+
+
+class ForgeMacroPattern(markdown.inlinepatterns.Pattern):
+
+ def __init__(self, *args, **kwargs):
+ self.ext = kwargs.pop('ext')
+ self.macro = macro.parse(self.ext._macro_context)
+ markdown.inlinepatterns.Pattern.__init__(self, *args, **kwargs)
+
+ def handleMatch(self, m):
+ html = self.macro(m.group(2))
+ placeholder = self.markdown.htmlStash.store(html)
+ return placeholder
+
+
+class ForgeLinkTreeProcessor(markdown.treeprocessors.Treeprocessor):
+
+ '''Wraps artifact links with []'''
+
+ def __init__(self, parent):
+ self.parent = parent
+ self.alinks = []
+
+ def run(self, root):
+ for node in root.getiterator('a'):
+ if 'alink' in node.get('class', '').split() and node.text:
+ node.text = '[' + node.text + ']'
+ return root
+
+ def reset(self):
+ self.alinks = []
+
+
+class MarkAsSafe(markdown.postprocessors.Postprocessor):
+
+ def run(self, text):
+ return h.html.literal(text)
+
+
+class AddCustomClass(markdown.postprocessors.Postprocessor):
+
+ def run(self, text):
+ return '<div class="markdown_content">%s</div>' % text
+
+
+class RelativeLinkRewriter(markdown.postprocessors.Postprocessor):
+
+ def __init__(self, make_absolute=False):
+ self._make_absolute = make_absolute
+
+ def run(self, text):
+ soup = BeautifulSoup(text)
+ if self._make_absolute:
+ rewrite = self._rewrite_abs
+ else:
+ rewrite = self._rewrite
+ for link in soup.findAll('a'):
+ rewrite(link, 'href')
+ for link in soup.findAll('img'):
+ rewrite(link, 'src')
+ # BeautifulSoup always stores data in unicode,
+ # but when doing unicode(soup) it does some strange things
+ # like nesting html comments, e.g. returns <!--<!-- comment -->-->
+ # instead of <!-- comment -->.
+ # Converting soup object to string representation first,
+ # and then back to unicode avoids that.
+ # str() called on BeautifulSoup document always returns string
+ # encoded in utf-8, so this should always work.
+ return h.really_unicode(str(soup))
+
+ def _rewrite(self, tag, attr):
+ val = tag.get(attr)
+ if val is None:
+ return
+ if ' ' in val:
+ # Don't urllib.quote to avoid possible double-quoting
+ # just make sure no spaces
+ val = val.replace(' ', '%20')
+ tag[attr] = val
+ if '://' in val:
+ for domain in re.split(r'\s*,\s*', config.get('nofollow_exempt_domains', '')):
+ if domain and domain in val:
+ return
+ tag['rel'] = 'nofollow'
+ return
+ if val.startswith('/'):
+ return
+ if val.startswith('.'):
+ return
+ if val.startswith('mailto:'):
+ return
+ if val.startswith('#'):
+ return
+ tag[attr] = '../' + val
+
+ def _rewrite_abs(self, tag, attr):
+ self._rewrite(tag, attr)
+ val = tag.get(attr)
+ val = urljoin(config['base_url'], val)
+ tag[attr] = val
+
+
+class HTMLSanitizer(markdown.postprocessors.Postprocessor):
+
+ def run(self, text):
+ parser = html5lib.HTMLParser(tokenizer=ForgeHTMLSanitizer)
+ parsed = parser.parse(text)
+ serializer = html5lib.serializer.HTMLSerializer()
+ walker = html5lib.getTreeWalker("etree")
+ stream = html5lib.filters.alphabeticalattributes.Filter(walker(parsed))
+ out = ''.join(serializer.serialize(stream))
+ return out
+
+
+class AutolinkPattern(markdown.inlinepatterns.Pattern):
+
+ def __init__(self, pattern, markdown_instance=None):
+ markdown.inlinepatterns.Pattern.__init__(
+ self, pattern, markdown_instance)
+ # override the complete regex, requiring the preceding text (.*?) to end
+ # with whitespace or beginning of line "\s|^"
+ self.compiled_re = re.compile("^(.*?\s|^)%s(.*?)$" % pattern,
+ re.DOTALL | re.UNICODE)
+
+ def handleMatch(self, mo):
+ old_link = mo.group(2)
+ result = markdown.util.etree.Element('a')
+ result.text = old_link
+ # since this is run before the builtin 'escape' processor, we have to
+ # do our own unescaping
+ for char in markdown.Markdown.ESCAPED_CHARS:
+ old_link = old_link.replace('\\' + char, char)
+ result.set('href', old_link)
+ return result
+
+
+class ForgeMacroIncludePreprocessor(markdown.preprocessors.Preprocessor):
+
+ '''Join include statements to prevent extra <br>'s inserted by nl2br extension.
+
+ Converts:
+ [[include ref=some_ref]]
+ [[include ref=some_other_ref]]
+
+ To:
+ [[include ref=some_ref]][[include ref=some_other_ref]]
+ '''
+ pattern = re.compile(r'^\s*\[\[include ref=[^\]]*\]\]\s*$', re.IGNORECASE)
+
+ def run(self, lines):
+ buf = []
+ result = []
+ for line in lines:
+ if self.pattern.match(line):
+ buf.append(line)
+ else:
+ if buf:
+ result.append(''.join(buf))
+ buf = []
+ result.append(line)
+ return result
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/lib/package_path_loader.py
----------------------------------------------------------------------
diff --git a/lib/package_path_loader.py b/lib/package_path_loader.py
new file mode 100644
index 0000000..d96800c
--- /dev/null
+++ b/lib/package_path_loader.py
@@ -0,0 +1,288 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+A Jinja template loader which allows for:
+
+- dotted-notation package loading
+- search-path-based overriding of same
+
+Dotted notation
+---------------
+
+- Allow a Tool implementer to use a dotted-notation module name
+ (as occuring in the ``PYTHONPATH``), then the given path within the
+ module::
+
+ @expose('jinja:<module.name>:<path/within/module.html>')
+
+ e.g.::
+
+ @expose('jinja:allura:templates/repo/file.html')
+
+Overriding dotted notation
+--------------------------
+
+Allow a Tool implementer to override the theme baseline (or any
+other Tool's) templates. This can be lighter-weight than subclassing
+:class:`allura.plugin.ThemeProvider`, plus will allow for more fine-grained
+changes.
+
+This will also override ``extends`` and ``import`` Jinja tags.
+
+This approach uses a:
+
+- ``setup.py`` entry point to a class with...
+- *magic* files and...
+- (optionally) a class property to specify ordering
+
+File Structure for Overriding dotted notation
+=============================================
+
+For the examples, assume the following directory structure::
+
+ NewTool/
+ |- setup.py <- entry point specified here
+ |- newtool/
+ |- app.py <- entry point target here
+ |- templates/
+ | |- index.html <- Tool's regular templates
+ |- override <- override_root
+ |- allura/ <- magic directory named after module
+ |- templates/
+ |- repo/
+ |- file.html <- actual template
+
+To override the above example, a Tool implementer would
+add the following line to their Tool's ``setup.py`` (assuming usage in Allura,
+with the default ``app_cfg``)::
+
+ [allura.theme.override]
+ newtool = newtool.app:NewToolApp
+
+Then, in the neighbor path (see below) for the file containing the
+Tool class, add the following path/file::
+
+ override/allura/templates/repo/file.html
+
+The template will be overridden. Note that after changing
+``setup.py``, it would be required to re-initialize with setuptools::
+
+ python setup.py develop
+
+Specifying search path order with template_path_rules
+=====================================================
+
+If a highly specific ordering is required, such as if multiple Tools
+are trying to override the same template, the entry point target
+class can also contain a class property template_path_rules::
+
+ class NewToolApp(Application):
+ template_path_rules = [
+ ['>', 'old-tool'],
+ ]
+
+Each rule specifies a postioner and an entry point or "signpost".
+If no rule is provided, the default is ``['>', 'allura']``.
+
+The "signposts" are:
+
+- Any other app's override entry point name
+- ``site-theme``
+- ``allura`` (you probably shouldn't do this)
+- ``project-theme`` **NOT IMPLEMENTED**
+- ``tool-theme`` **NOT IMPLEMENTED**
+
+The positioners are:
+
+ >
+ This overrider will be found BEFORE the specified entry point
+
+ <
+ This overrider will be found AFTER the specified entry point
+
+ =
+ This will replace one of the "signpost" entry points (if multiple apps
+ try to do this for the same signpost, the result is undefined)
+
+**TODO:** Support multiple partial themes
+
+"""
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+import pkg_resources
+import os
+
+import jinja2
+from tg import config
+from paste.deploy.converters import asbool
+from ming.utils import LazyProperty
+
+from allura.lib.helpers import topological_sort, iter_entry_points
+
+
+class PackagePathLoader(jinja2.BaseLoader):
+
+ def __init__(self, override_entrypoint='allura.theme.override',
+ default_paths=None,
+ override_root='override',
+ ):
+ '''
+ Set up initial values... defaults are for Allura.
+ '''
+ # TODO: How does one handle project-theme?
+ if default_paths is None:
+ default_paths = [
+ #['project-theme', None],
+ ['site-theme', None],
+ ['allura', '/'],
+ ]
+
+ self.override_entrypoint = override_entrypoint
+ self.default_paths = default_paths
+ self.override_root = override_root
+
+ @LazyProperty
+ def fs_loader(self):
+ return jinja2.FileSystemLoader(self.init_paths())
+
+ def _load_paths(self):
+ """
+ Load all the paths to be processed, including defaults, in the default order.
+ """
+ paths = self.default_paths[:] # copy default_paths
+ paths[-1:0] = [ # insert all eps just before last item, by default
+ [ep.name, pkg_resources.resource_filename(ep.module_name, "")]
+ for ep in iter_entry_points(self.override_entrypoint)
+ ]
+ return paths
+
+ def _load_rules(self):
+ """
+ Load and pre-process the rules from the entry points.
+
+ Rules are specified per-tool as a list of the form:
+
+ template_path_rules = [
+ ['>', 'tool1'], # this tool must be resolved before tool1
+ ['<', 'tool2'], # this tool must be resolved after tool2
+ ['=', 'tool3'], # this tool replaces all of tool3's templates
+ ]
+
+ Returns two lists of rules, order_rules and replacement_rules.
+
+ order_rules represents all of the '>' and '<' rules and are returned
+ as a list of pairs of the form ('a', 'b') indicating that path 'a' must
+ come before path 'b'.
+
+ replacement_rules represent all of the '=' rules and are returned as
+ a dictionary mapping the paths to replace to the paths to replace with.
+ """
+ order_rules = []
+ replacement_rules = {}
+ for ep in iter_entry_points(self.override_entrypoint):
+ for rule in getattr(ep.load(), 'template_path_rules', []):
+ if rule[0] == '>':
+ order_rules.append((ep.name, rule[1]))
+ elif rule[0] == '=':
+ replacement_rules[rule[1]] = ep.name
+ elif rule[0] == '<':
+ order_rules.append((rule[1], ep.name))
+ else:
+ raise jinja2.TemplateError(
+ 'Unknown template path rule in %s: %s' % (
+ ep.name, ' '.join(rule)))
+ return order_rules, replacement_rules
+
+ def _sort_paths(self, paths, rules):
+ """
+ Process all '>' and '<' rules, providing a partial ordering
+ of the paths based on the given rules.
+
+ The rules should already have been pre-processed by _load_rules
+ to a list of partial ordering pairs ('a', 'b') indicating that
+ path 'a' should come before path 'b'.
+ """
+ names = [p[0] for p in paths]
+ # filter rules that reference non-existent paths to prevent "loops" in
+ # the graph
+ rules = [r for r in rules if r[0] in names and r[1] in names]
+ ordered_paths = topological_sort(names, rules)
+ if ordered_paths is None:
+ raise jinja2.TemplateError(
+ 'Loop detected in ordering of overrides')
+ return paths.sort(key=lambda p: ordered_paths.index(p[0]))
+
+ def _replace_signposts(self, paths, rules):
+ """
+ Process all '=' rules, replacing the rule target's path value with
+ the rule's entry's path value.
+
+ Multiple entries replacing the same signpost can cause indeterminate
+ behavior, as the order of the entries is not entirely defined.
+ However, if _sort_by_rules is called first, the partial ordering is
+ respected.
+
+ This mutates paths.
+ """
+ p_idx = lambda n: [e[0] for e in paths].index(n)
+ for target, replacement in list(rules.items()):
+ try:
+ removed = paths.pop(p_idx(replacement))
+ paths[p_idx(target)][1] = removed[1]
+ except ValueError:
+ # target or replacement missing (may not be installed)
+ pass
+
+ def init_paths(self):
+ '''
+ Set up the setuptools entry point-based paths.
+ '''
+ paths = self._load_paths()
+ order_rules, repl_rules = self._load_rules()
+
+ self._sort_paths(paths, order_rules)
+ self._replace_signposts(paths, repl_rules)
+
+ return [p[1] for p in paths if p[1] is not None]
+
+ def get_source(self, environment, template):
+ '''
+ Returns the source for jinja2 rendered templates. Can understand...
+ - path/to/template.html
+ - module:path/to/template.html
+ '''
+ # look in all of the customized search locations...
+ if not asbool(config.get('disable_template_overrides', False)):
+ try:
+ parts = [self.override_root] + template.split(':')
+ if len(parts) > 2:
+ parts[1:2] = parts[1].split('.')
+ return self.fs_loader.get_source(environment,
+ os.path.join(*parts))
+ except jinja2.TemplateNotFound:
+ # fall-back to attempt non-override loading
+ pass
+
+ if ':' in template:
+ package, path = template.split(':', 2)
+ filename = pkg_resources.resource_filename(package, path)
+ return self.fs_loader.get_source(environment, filename)
+ else:
+ return self.fs_loader.get_source(environment, template)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/lib/patches.py
----------------------------------------------------------------------
diff --git a/lib/patches.py b/lib/patches.py
new file mode 100644
index 0000000..63225e7
--- /dev/null
+++ b/lib/patches.py
@@ -0,0 +1,127 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+
+import webob
+import tg.decorators
+from decorator import decorator
+from pylons import request
+import mock
+import simplejson
+
+from allura.lib import helpers as h
+
+_patched = False
+def apply():
+ global _patched
+ if _patched:
+ return
+ _patched = True
+
+ old_lookup_template_engine = tg.decorators.Decoration.lookup_template_engine
+
+ @h.monkeypatch(tg.decorators.Decoration)
+ def lookup_template_engine(self, request):
+ '''Wrapper to handle totally borked-up HTTP-ACCEPT headers'''
+ try:
+ return old_lookup_template_engine(self, request)
+ except:
+ pass
+ environ = dict(request.environ, HTTP_ACCEPT='*/*')
+ request = webob.Request(environ)
+ return old_lookup_template_engine(self, request)
+
+ @h.monkeypatch(tg, tg.decorators)
+ def override_template(controller, template):
+ '''Copy-pasted patch to allow multiple colons in a template spec'''
+ if hasattr(controller, 'decoration'):
+ decoration = controller.decoration
+ else:
+ return
+ if hasattr(decoration, 'engines'):
+ engines = decoration.engines
+ else:
+ return
+
+ for content_type, content_engine in engines.items():
+ template = template.split(':', 1)
+ template.extend(content_engine[2:])
+ try:
+ override_mapping = request._override_mapping
+ except AttributeError:
+ override_mapping = request._override_mapping = {}
+ override_mapping[controller.__func__] = {content_type: template}
+
+ @h.monkeypatch(tg, tg.decorators)
+ @decorator
+ def without_trailing_slash(func, *args, **kwargs):
+ '''Monkey-patched to use 301 redirects for SEO'''
+ response_type = getattr(request, 'response_type', None)
+ if (request.method == 'GET' and request.path.endswith('/')
+ and not response_type and len(request.params) == 0):
+ raise webob.exc.HTTPMovedPermanently(location=request.url[:-1])
+ return func(*args, **kwargs)
+
+ @h.monkeypatch(tg, tg.decorators)
+ @decorator
+ def with_trailing_slash(func, *args, **kwargs):
+ '''Monkey-patched to use 301 redirects for SEO'''
+ response_type = getattr(request, 'response_type', None)
+ if (request.method == 'GET' and not(request.path.endswith('/'))
+ and not response_type and len(request.params) == 0):
+ raise webob.exc.HTTPMovedPermanently(location=request.url + '/')
+ return func(*args, **kwargs)
+
+ # http://blog.watchfire.com/wfblog/2011/10/json-based-xss-exploitation.html
+ # change < to its unicode escape when rendering JSON out of turbogears
+ # This is to avoid IE9 and earlier, which don't know the json content type
+ # and may attempt to render JSON data as HTML if the URL ends in .html
+ original_tg_jsonify_GenericJSON_encode = tg.jsonify.GenericJSON.encode
+ escape_pattern_with_lt = re.compile(
+ simplejson.encoder.ESCAPE.pattern.rstrip(']') + '<' + ']')
+
+ @h.monkeypatch(tg.jsonify.GenericJSON)
+ def encode(self, o):
+ # ensure_ascii=False forces encode_basestring() to be called instead of
+ # encode_basestring_ascii() and encode_basestring_ascii may likely be c-compiled
+ # and thus not monkeypatchable
+ with h.push_config(self, ensure_ascii=False), \
+ h.push_config(simplejson.encoder, ESCAPE=escape_pattern_with_lt), \
+ mock.patch.dict(simplejson.encoder.ESCAPE_DCT, {'<': r'\u003C'}):
+ return original_tg_jsonify_GenericJSON_encode(self, o)
+
+
+# must be saved outside the newrelic() method so that multiple newrelic()
+# calls (e.g. during tests) don't cause the patching to get applied to itself
+# over and over
+old_controller_call = tg.controllers.DecoratedController._call
+
+
+def newrelic():
+ @h.monkeypatch(tg.controllers.DecoratedController,
+ tg.controllers.decoratedcontroller.DecoratedController)
+ def _call(self, controller, *args, **kwargs):
+ '''Set NewRelic transaction name to actual controller name'''
+ import newrelic.agent
+ newrelic.agent.set_transaction_name(
+ newrelic.agent.callable_name(controller))
+ return old_controller_call(self, controller, *args, **kwargs)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/lib/phone/__init__.py
----------------------------------------------------------------------
diff --git a/lib/phone/__init__.py b/lib/phone/__init__.py
new file mode 100644
index 0000000..cbe5cd9
--- /dev/null
+++ b/lib/phone/__init__.py
@@ -0,0 +1,78 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class PhoneService(object):
+ """
+ Defines the phone verification service interface and provides a default
+ no-op implementation.
+ """
+
+ def __init__(self, config):
+ pass
+
+ def verify(self, number):
+ """
+ Generate PIN and send it to phone number :param number:
+
+ Returns dict with following keys: status, request_id, error. status is
+ either 'ok' or 'error'.
+
+ If status is 'ok' then request_id is present (used later to verify PIN)
+ otherwise 'error' is an error message.
+ """
+ log.info('Phone service is not configured')
+ return {
+ 'status': 'error',
+ 'error': 'Phone service is not configured',
+ }
+
+ def check(self, request_id, pin):
+ """
+ Given the :param pin: code user entered and :param request_id:
+ (obtained by verify), verify that :param pin: is valid.
+
+ Returns dict with following keys: status, error. status is either 'ok'
+ or 'error'.
+
+ If status is 'ok' then verification was successful otherwise 'error' is
+ an error message.
+ """
+ log.info('Phone service is not configured')
+ return {
+ 'status': 'error',
+ 'error': 'Phone service is not configured',
+ }
+
+ @classmethod
+ def get(cls, config, entry_points):
+ """
+ Return an instance of PhoneService implementation based on ``config``.
+ """
+ method = config.get('phone.method')
+ if not method:
+ return cls(config)
+ service = entry_points[method]
+ return service(config)
http://git-wip-us.apache.org/repos/asf/allura/blob/d52f8e2a/lib/phone/nexmo.py
----------------------------------------------------------------------
diff --git a/lib/phone/nexmo.py b/lib/phone/nexmo.py
new file mode 100644
index 0000000..13c2051
--- /dev/null
+++ b/lib/phone/nexmo.py
@@ -0,0 +1,93 @@
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import unicode_literals
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+from urllib.parse import urljoin
+
+import json
+import requests
+from allura.lib.phone import PhoneService
+from allura.lib.utils import phone_number_hash
+
+log = logging.getLogger(__name__)
+
+
+class NexmoPhoneService(PhoneService):
+ """
+ Implementation of :class:`allura.lib.phone.PhoneService` interface
+ for Nexmo Verify
+ """
+
+ BASE_URL = 'https://api.nexmo.com/'
+
+ def __init__(self, config):
+ self.config = config
+ self.api_key = config.get('phone.api_key')
+ self.api_secret = config.get('phone.api_secret')
+
+ def add_common_params(self, params):
+ common = {
+ 'api_key': self.api_key,
+ 'api_secret': self.api_secret,
+ }
+ return dict(params, **common)
+
+ def error(self, msg):
+ return {'status': 'error', 'error': msg}
+
+ def ok(self, **params):
+ return dict({'status': 'ok'}, **params)
+
+ def post(self, url, **params):
+ if url[-1] != '/':
+ url += '/'
+ url = urljoin(url, 'json')
+ headers = {'Content-Type': 'application/json'}
+ params = self.add_common_params(params)
+ log_params = dict(params, api_key='...', api_secret='...')
+ if 'number' in log_params:
+ log_params['number'] = phone_number_hash(log_params['number'])
+ params = json.dumps(params, sort_keys=True)
+ log.info('PhoneService (nexmo) request: %s %s', url, log_params)
+ try:
+ resp = requests.post(url, data=params, headers=headers)
+ log.info('PhoneService (nexmo) response: %s', resp.content)
+ resp = resp.json()
+ except Exception:
+ msg = 'Failed sending request to Nexmo'
+ log.exception(msg)
+ return self.error(msg)
+ if resp.get('status') == '0':
+ return self.ok(request_id=resp.get('request_id'))
+ return self.error(resp.get('error_text'))
+
+ def verify(self, number):
+ url = urljoin(self.BASE_URL, 'verify')
+ # Required. Brand or name of your app, service the verification is
+ # for. This alphanumeric (maximum length 18 characters) will be
+ # used inside the body of all SMS and TTS messages sent (e.g. "Your
+ # <brand> PIN code is ..")
+ brand = self.config.get('site_name')[:18]
+ return self.post(url, number=number, brand=brand)
+
+ def check(self, request_id, pin):
+ url = urljoin(self.BASE_URL, 'verify/check')
+ return self.post(url, request_id=request_id, code=pin)