You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2012/11/07 22:00:48 UTC
[6/8] git commit: [#4667] ticket:183 rewrite ForgeExtension for
markdown
[#4667] ticket:183 rewrite ForgeExtension for markdown
* change links and macros to inline pattrens
* small fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/e4328720
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/e4328720
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/e4328720
Branch: refs/heads/master
Commit: e432872035c5988f1971e63d85d9caa3da1b99be
Parents: 31b8b50
Author: Igor Bondarenko <je...@gmail.com>
Authored: Thu Oct 4 15:27:50 2012 +0300
Committer: Dave Brondsema <db...@geek.net>
Committed: Wed Nov 7 20:30:56 2012 +0000
----------------------------------------------------------------------
Allura/allura/lib/markdown_extensions.py | 202 ++++++++++---------------
Allura/allura/lib/search.py | 4 +-
Allura/allura/tests/test_globals.py | 26 +++-
3 files changed, 105 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/e4328720/Allura/allura/lib/markdown_extensions.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/markdown_extensions.py b/Allura/allura/lib/markdown_extensions.py
index 6156b62..68741ca 100644
--- a/Allura/allura/lib/markdown_extensions.py
+++ b/Allura/allura/lib/markdown_extensions.py
@@ -1,15 +1,9 @@
import re
-import os
import logging
-import string
-from collections import defaultdict
-from urllib import quote
from urlparse import urljoin
-from ConfigParser import RawConfigParser
-from pprint import pformat
from tg import config
-from pylons import c, g, request
+from pylons import request
from BeautifulSoup import BeautifulSoup
import markdown
@@ -17,6 +11,7 @@ import feedparser
from . import macro
from . import helpers as h
+from allura import model as M
log = logging.getLogger(__name__)
@@ -25,6 +20,8 @@ PLAINTEXT_BLOCK_RE = re.compile( \
re.MULTILINE|re.DOTALL
)
+MACRO_PATTERN = r'\[\[([^\]\[]+)\]\]'
+
class ForgeExtension(markdown.Extension):
@@ -36,13 +33,16 @@ class ForgeExtension(markdown.Extension):
def extendMarkdown(self, md, md_globals):
md.registerExtension(self)
- self.forge_processor = ForgeProcessor(self._use_wiki, md, macro_context=self._macro_context)
- self.forge_processor.install()
md.preprocessors['fenced-code'] = FencedCodeProcessor()
md.preprocessors.add('plain_text_block', PlainTextPreprocessor(md), "_begin")
md.inlinePatterns['autolink_1'] = AutolinkPattern(r'(http(?:s?)://[a-zA-Z0-9./\-_0%?&=+#;~:]+)')
# replace the link pattern with our extended version
- md.inlinePatterns['link'] = ForgeLinkPattern(markdown.inlinepatterns.LINK_RE, md)
+ md.inlinePatterns['link'] = ForgeLinkPattern(markdown.inlinepatterns.LINK_RE, md, ext=self)
+ md.inlinePatterns['short_reference'] = ForgeLinkPattern(markdown.inlinepatterns.SHORT_REF_RE, md, ext=self)
+ # macro must be processed before links
+ md.inlinePatterns.add('macro', ForgeMacroPattern(MACRO_PATTERN, md, ext=self), '<link')
+ self.forge_link_tree_processor = ForgeLinkTreeProcessor(md)
+ md.treeprocessors['links'] = self.forge_link_tree_processor
md.treeprocessors['br'] = LineOrientedTreeProcessor(md)
# Sanitize HTML
md.postprocessors['sanitize_html'] = HTMLSanitizer()
@@ -54,19 +54,59 @@ class ForgeExtension(markdown.Extension):
md.postprocessors['mark_safe'] = MarkAsSafe()
def reset(self):
- self.forge_processor.reset()
+ self.forge_link_tree_processor.reset()
+
class ForgeLinkPattern(markdown.inlinepatterns.LinkPattern):
artifact_re = re.compile(r'((.*?):)?((.*?):)?(.+)')
- def sanitize_url(self, url):
- url = markdown.inlinepatterns.Pattern.sanitize_url(url)
- log.warn('url is %s' % url)
- log.info('url is %s' % url)
- if not url and self.artifact_re.match(url):
- url = 'LINK %s LINK' % url
- return url
+ def __init__(self, *args, **kwargs):
+ self.ext = kwargs.pop('ext')
+ markdown.inlinepatterns.LinkPattern.__init__(self, *args, **kwargs)
+
+ def handleMatch(self, m):
+ el = markdown.util.etree.Element('a')
+ el.text = m.group(2)
+ try:
+ href = m.group(9)
+ except IndexError:
+ href = m.group(2)
+ try:
+ title = m.group(13)
+ except IndexError:
+ title = None
+
+ if href:
+ if href == 'TOC':
+ return '[TOC]' # skip TOC
+ if self.artifact_re.match(href):
+ href, classes = self._expand_alink(href)
+ el.set('href', self.sanitize_url(self.unescape(href.strip())))
+ el.set('class', classes)
+ else:
+ el.set('href', '')
+
+ if title:
+ title = markdown.inlinepatterns.dequote(self.unescape(title))
+ el.set('title', title)
+
+ return el
+
+ def _expand_alink(self, link):
+ '''Return (href, classes) for an artifact link'''
+ classes = ''
+ href = link
+ shortlink = M.Shortlink.lookup(link)
+ if shortlink:
+ href = shortlink.url
+ classes = 'alink'
+ self.ext.forge_link_tree_processor.alinks.append(link)
+ elif self.ext._use_wiki and ':' not in link:
+ href = h.urlquote(link)
+ classes = 'notfound alink'
+ return href, classes
+
class PlainTextPreprocessor(markdown.preprocessors.Preprocessor):
'''
@@ -96,6 +136,7 @@ class PlainTextPreprocessor(markdown.preprocessors.Preprocessor):
txt = txt.replace('"', '"')
return txt
+
class FencedCodeProcessor(markdown.preprocessors.Preprocessor):
pattern = '~~~~'
@@ -112,133 +153,49 @@ class FencedCodeProcessor(markdown.preprocessors.Preprocessor):
new_lines.append(line)
return new_lines
-class ForgeProcessor(object):
- macro_pattern = r'\[(\[([^\]\[]*)\])\]'
- placeholder_prefix = '#jgimwge'
- placeholder = '%s:%%s:%%.4d#khjhhj' % placeholder_prefix
- placeholder_re = re.compile('%s:(\\w+):(\\d+)#khjhhj' % placeholder_prefix)
-
- def __init__(self, use_wiki = False, markdown=None, macro_context=None):
- self.markdown = markdown
- self._use_wiki = use_wiki
- self._macro_context = macro_context
- self.inline_patterns = {
- 'forge.macro' : ForgeInlinePattern(self, self.macro_pattern)}
- self.tree_processor = ForgeTreeProcessor(self)
- self.reset()
- self.artifact_re = re.compile(r'((.*?):)?((.*?):)?(.+)')
- self.macro_re = re.compile(self.alink_pattern) # BUG
-
- def install(self):
- for k,v in self.inline_patterns.iteritems():
- self.markdown.inlinePatterns[k] = v
- if self._use_wiki:
- self.markdown.treeprocessors['forge'] = self.tree_processor
- self.markdown.postprocessors['forge'] = self.postprocessor
-
- def store(self, raw):
- if self.macro_re.match(raw):
- stash = 'macro'
- raw = raw[1:-1] # strip off the enclosing []
- elif raw == 'TOC':
- # pass this straight through and don't do an artifact lookup
- return '[' + raw + ']'
- elif self.artifact_re.match(raw):
- stash = 'artifact'
- else:
- return raw
- print 'store', stash, raw
- return self._store(stash, raw)
-
- def _store(self, stash_name, value):
- placeholder = self.placeholder % (stash_name, len(self.stash[stash_name]))
- self.stash[stash_name].append(value)
- return placeholder
-
- def lookup(self, stash, id):
- stash = self.stash.get(stash, [])
- if id >= len(stash):
- return ''
- print 'lookup returning', stash[id]
- return stash[id]
- def compile(self):
- from allura import model as M
- if self.stash['artifact'] or self.stash['link']:
- try:
- self.alinks = M.Shortlink.from_links(*self.stash['artifact'])
- self.alinks.update(M.Shortlink.from_links(*self.stash['link']))
- except:
- self.alinks = {}
- self.stash['artifact'] = map(self._expand_alink, self.stash['artifact'])
- self.stash['link'] = map(self._expand_link, self.stash['link'])
- print 'pre:', self.stash['macro']
- self.stash['macro'] = map(macro.parse(self._macro_context), self.stash['macro'])
- print 'post:', self.stash['macro']
-
- def reset(self):
- self.stash = dict(
- artifact=[],
- macro=[],
- link=[])
- self.alinks = {}
- self.compiled = False
-
- def _expand_alink(self, link):
- new_link = self.alinks.get(link, None)
- if new_link:
- return '<a href="%s">[%s]</a>' % (
- new_link.url, link)
- elif self._use_wiki and ':' not in link:
- return '<a href="%s" class="notfound">[%s]</a>' % (
- h.urlquote(link), link)
- else:
- return link
-
- def _expand_link(self, link):
- reference = self.alinks.get(link)
- mailto = u'\x02amp\x03#109;\x02amp\x03#97;\x02amp\x03#105;\x02amp\x03#108;\x02amp\x03#116;\x02amp\x03#111;\x02amp\x03#58;'
- if not reference and not link.startswith(mailto) and '#' not in link:
- return 'notfound'
- else:
- return ''
+class ForgeMacroPattern(markdown.inlinepatterns.Pattern):
-class ForgeInlinePattern(markdown.inlinepatterns.Pattern):
-
- def __init__(self, parent, pattern):
- self.parent = parent
- markdown.inlinepatterns.Pattern.__init__(
- self, pattern, parent.markdown)
+ def __init__(self, *args, **kwargs):
+ self.ext = kwargs.pop('ext')
+ self.macro = macro.parse(self.ext._macro_context)
+ markdown.inlinepatterns.Pattern.__init__(self, *args, **kwargs)
def handleMatch(self, m):
- return self.parent.store(m.group(2))
+ html = self.macro(m.group(2))
+ placeholder = self.markdown.htmlStash.store(html)
+ return placeholder
-class ForgeTreeProcessor(markdown.treeprocessors.Treeprocessor):
- '''This flags intra-wiki links that point to non-existent pages'''
+class ForgeLinkTreeProcessor(markdown.treeprocessors.Treeprocessor):
+ '''Wraps artifact links with []'''
def __init__(self, parent):
self.parent = parent
+ self.alinks = []
def run(self, root):
for node in root.getiterator('a'):
- href = node.get('href')
- if not href: continue
- if '/' in href: continue
- classes = node.get('class', '').split() + [ self.parent._store('link', href) ]
- node.attrib['class'] = ' '.join(classes)
+ if 'alink' in node.get('class', '').split():
+ node.text = '[' + node.text + ']'
return root
+ def reset(self):
+ self.alinks = []
+
+
class MarkAsSafe(markdown.postprocessors.Postprocessor):
def run(self, text):
return h.html.literal(text)
+
class AddCustomClass(markdown.postprocessors.Postprocessor):
def run(self, text):
return '<div class="markdown_content">%s</div>' % text
+
class RelativeLinkRewriter(markdown.postprocessors.Postprocessor):
def __init__(self, make_absolute=False):
@@ -287,6 +244,7 @@ class RelativeLinkRewriter(markdown.postprocessors.Postprocessor):
val = urljoin(config.get('base_url', 'http://sourceforge.net/'),val)
tag[attr] = val
+
class HTMLSanitizer(markdown.postprocessors.Postprocessor):
def run(self, text):
@@ -297,6 +255,7 @@ class HTMLSanitizer(markdown.postprocessors.Postprocessor):
p.feed(text.encode('utf-8'))
return unicode(p.output(), 'utf-8')
+
class LineOrientedTreeProcessor(markdown.treeprocessors.Treeprocessor):
'''Once MD is satisfied with the etree, this runs to replace \n with <br/>
within <p>s.
@@ -331,6 +290,7 @@ class LineOrientedTreeProcessor(markdown.treeprocessors.Treeprocessor):
node[:] = list(new_node)
return root
+
class AutolinkPattern(markdown.inlinepatterns.LinkPattern):
def handleMatch(self, mo):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/e4328720/Allura/allura/lib/search.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/search.py b/Allura/allura/lib/search.py
index 3870a78..7fe7388 100644
--- a/Allura/allura/lib/search.py
+++ b/Allura/allura/lib/search.py
@@ -65,5 +65,5 @@ def find_shortlinks(text):
extensions=['codehilite', ForgeExtension(), 'tables'],
output_format='html4')
md.convert(text)
- link_index = md.postprocessors['forge'].parent.alinks
- return [ link for link in link_index.itervalues() if link is not None]
+ link_index = md.treeprocessors['links'].alinks
+ return [ link for link in link_index if link is not None]
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/e4328720/Allura/allura/tests/test_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_globals.py b/Allura/allura/tests/test_globals.py
index b528cab..2a28b34 100644
--- a/Allura/allura/tests/test_globals.py
+++ b/Allura/allura/tests/test_globals.py
@@ -155,7 +155,22 @@ def test_markdown_toc():
# Header 1
## Header 2""")
- assert '<a class="" href="#header-1">Header 1</a>' in r, r
+ assert '''<ul>
+<li><a href="#header-1">Header 1</a><ul>
+<li><a href="#header-2">Header 2</a></li>
+</ul>
+</li>
+</ul>''' in r, r
+
+@with_setup(setUp)
+def test_markdown_links():
+ h.set_context('test', 'wiki', neighborhood='Projects')
+ text = g.markdown.convert('Read [here](Home) about our project')
+ assert '<a class="alink" href="/p/test/wiki/Home/">[here]</a>' in text, text
+ text = g.markdown.convert('[Go home](test:wiki:Home)')
+ assert '<a class="alink" href="/p/test/wiki/Home/">[Go home]</a>' in text, text
+ text = g.markdown.convert('See [test:wiki:Home]')
+ assert '<a class="alink" href="/p/test/wiki/Home/">[test:wiki:Home]</a>' in text, text
def test_markdown_xml_error():
r = g.markdown_wiki.convert(""" <?xml version="1.0" encoding="UTF-8"?>
@@ -166,9 +181,12 @@ def test_markdown_xml_error():
def test_markdown():
'Just a test to get coverage in our markdown extension'
h.set_context('test', 'wiki', neighborhood='Projects')
- assert '<a href=' in g.markdown.convert('# Foo!\n[Home]')
- assert '<a href=' not in g.markdown.convert('# Foo!\n[Rooted]')
- assert '<a href=' in g.markdown.convert('This is http://sf.net')
+ text = g.markdown.convert('# Foo!\n[Home]')
+ assert '<a class="alink" href=' in text, text
+ text = g.markdown.convert('# Foo!\n[Rooted]')
+ assert '<a href=' not in text, text
+ text = g.markdown.convert('This is http://sf.net')
+ assert '<a href=' in text, text
tgt = 'http://everything2.com/?node=nate+oostendorp'
s = g.markdown.convert('This is %s' % tgt)
assert_equal(