You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/09/03 20:20:03 UTC
[18/22] AMBARI-7138. Ambari RPM deals with jinja2 dependency
incorrectly (aonishuk)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c3ea59f/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py
new file mode 100644
index 0000000..bdb545e
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py
@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+"""
+ ambari_jinja2.filters
+ ~~~~~~~~~~~~~~
+
+ Bundled jinja filters.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+import math
+from random import choice
+from operator import itemgetter
+from itertools import imap, groupby
+from ambari_jinja2.utils import Markup, escape, pformat, urlize, soft_unicode
+from ambari_jinja2.runtime import Undefined
+from ambari_jinja2.exceptions import FilterArgumentError, SecurityError
+
+
+_word_re = re.compile(r'\w+(?u)')
+
+
+def contextfilter(f):
+ """Decorator for marking context dependent filters. The current
+ :class:`Context` will be passed as first argument.
+ """
+ f.contextfilter = True
+ return f
+
+
+def evalcontextfilter(f):
+ """Decorator for marking eval-context dependent filters. An eval
+ context object is passed as first argument. For more information
+ about the eval context, see :ref:`eval-context`.
+
+ .. versionadded:: 2.4
+ """
+ f.evalcontextfilter = True
+ return f
+
+
+def environmentfilter(f):
+ """Decorator for marking evironment dependent filters. The current
+ :class:`Environment` is passed to the filter as first argument.
+ """
+ f.environmentfilter = True
+ return f
+
+
+def do_forceescape(value):
+ """Enforce HTML escaping. This will probably double escape variables."""
+ if hasattr(value, '__html__'):
+ value = value.__html__()
+ return escape(unicode(value))
+
+
+@evalcontextfilter
+def do_replace(eval_ctx, s, old, new, count=None):
+ """Return a copy of the value with all occurrences of a substring
+ replaced with a new one. The first argument is the substring
+ that should be replaced, the second is the replacement string.
+ If the optional third argument ``count`` is given, only the first
+ ``count`` occurrences are replaced:
+
+ .. sourcecode:: jinja
+
+ {{ "Hello World"|replace("Hello", "Goodbye") }}
+ -> Goodbye World
+
+ {{ "aaaaargh"|replace("a", "d'oh, ", 2) }}
+ -> d'oh, d'oh, aaargh
+ """
+ if count is None:
+ count = -1
+ if not eval_ctx.autoescape:
+ return unicode(s).replace(unicode(old), unicode(new), count)
+ if hasattr(old, '__html__') or hasattr(new, '__html__') and \
+ not hasattr(s, '__html__'):
+ s = escape(s)
+ else:
+ s = soft_unicode(s)
+ return s.replace(soft_unicode(old), soft_unicode(new), count)
+
+
+def do_upper(s):
+ """Convert a value to uppercase."""
+ return soft_unicode(s).upper()
+
+
+def do_lower(s):
+ """Convert a value to lowercase."""
+ return soft_unicode(s).lower()
+
+
+@evalcontextfilter
+def do_xmlattr(_eval_ctx, d, autospace=True):
+ """Create an SGML/XML attribute string based on the items in a dict.
+ All values that are neither `none` nor `undefined` are automatically
+ escaped:
+
+ .. sourcecode:: html+jinja
+
+ <ul{{ {'class': 'my_list', 'missing': none,
+ 'id': 'list-%d'|format(variable)}|xmlattr }}>
+ ...
+ </ul>
+
+ Results in something like this:
+
+ .. sourcecode:: html
+
+ <ul class="my_list" id="list-42">
+ ...
+ </ul>
+
+ As you can see it automatically prepends a space in front of the item
+ if the filter returned something unless the second parameter is false.
+ """
+ rv = u' '.join(
+ u'%s="%s"' % (escape(key), escape(value))
+ for key, value in d.iteritems()
+ if value is not None and not isinstance(value, Undefined)
+ )
+ if autospace and rv:
+ rv = u' ' + rv
+ if _eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv
+
+
+def do_capitalize(s):
+ """Capitalize a value. The first character will be uppercase, all others
+ lowercase.
+ """
+ return soft_unicode(s).capitalize()
+
+
+def do_title(s):
+ """Return a titlecased version of the value. I.e. words will start with
+ uppercase letters, all remaining characters are lowercase.
+ """
+ return soft_unicode(s).title()
+
+
+def do_dictsort(value, case_sensitive=False, by='key'):
+ """Sort a dict and yield (key, value) pairs. Because python dicts are
+ unsorted you may want to use this function to order them by either
+ key or value:
+
+ .. sourcecode:: jinja
+
+ {% for item in mydict|dictsort %}
+ sort the dict by key, case insensitive
+
+ {% for item in mydict|dicsort(true) %}
+ sort the dict by key, case sensitive
+
+ {% for item in mydict|dictsort(false, 'value') %}
+ sort the dict by key, case insensitive, sorted
+ normally and ordered by value.
+ """
+ if by == 'key':
+ pos = 0
+ elif by == 'value':
+ pos = 1
+ else:
+ raise FilterArgumentError('You can only sort by either '
+ '"key" or "value"')
+ def sort_func(item):
+ value = item[pos]
+ if isinstance(value, basestring) and not case_sensitive:
+ value = value.lower()
+ return value
+
+ return sorted(value.items(), key=sort_func)
+
+
+def do_sort(value, reverse=False, case_sensitive=False):
+ """Sort an iterable. Per default it sorts ascending, if you pass it
+ true as first argument it will reverse the sorting.
+
+ If the iterable is made of strings the third parameter can be used to
+ control the case sensitiveness of the comparison which is disabled by
+ default.
+
+ .. sourcecode:: jinja
+
+ {% for item in iterable|sort %}
+ ...
+ {% endfor %}
+ """
+ if not case_sensitive:
+ def sort_func(item):
+ if isinstance(item, basestring):
+ item = item.lower()
+ return item
+ else:
+ sort_func = None
+ return sorted(value, key=sort_func, reverse=reverse)
+
+
+def do_default(value, default_value=u'', boolean=False):
+ """If the value is undefined it will return the passed default value,
+ otherwise the value of the variable:
+
+ .. sourcecode:: jinja
+
+ {{ my_variable|default('my_variable is not defined') }}
+
+ This will output the value of ``my_variable`` if the variable was
+ defined, otherwise ``'my_variable is not defined'``. If you want
+ to use default with variables that evaluate to false you have to
+ set the second parameter to `true`:
+
+ .. sourcecode:: jinja
+
+ {{ ''|default('the string was empty', true) }}
+ """
+ if (boolean and not value) or isinstance(value, Undefined):
+ return default_value
+ return value
+
+
+@evalcontextfilter
+def do_join(eval_ctx, value, d=u''):
+ """Return a string which is the concatenation of the strings in the
+ sequence. The separator between elements is an empty string per
+ default, you can define it with the optional parameter:
+
+ .. sourcecode:: jinja
+
+ {{ [1, 2, 3]|join('|') }}
+ -> 1|2|3
+
+ {{ [1, 2, 3]|join }}
+ -> 123
+ """
+ # no automatic escaping? joining is a lot eaiser then
+ if not eval_ctx.autoescape:
+ return unicode(d).join(imap(unicode, value))
+
+ # if the delimiter doesn't have an html representation we check
+ # if any of the items has. If yes we do a coercion to Markup
+ if not hasattr(d, '__html__'):
+ value = list(value)
+ do_escape = False
+ for idx, item in enumerate(value):
+ if hasattr(item, '__html__'):
+ do_escape = True
+ else:
+ value[idx] = unicode(item)
+ if do_escape:
+ d = escape(d)
+ else:
+ d = unicode(d)
+ return d.join(value)
+
+ # no html involved, to normal joining
+ return soft_unicode(d).join(imap(soft_unicode, value))
+
+
+def do_center(value, width=80):
+ """Centers the value in a field of a given width."""
+ return unicode(value).center(width)
+
+
+@environmentfilter
+def do_first(environment, seq):
+ """Return the first item of a sequence."""
+ try:
+ return iter(seq).next()
+ except StopIteration:
+ return environment.undefined('No first item, sequence was empty.')
+
+
+@environmentfilter
+def do_last(environment, seq):
+ """Return the last item of a sequence."""
+ try:
+ return iter(reversed(seq)).next()
+ except StopIteration:
+ return environment.undefined('No last item, sequence was empty.')
+
+
+@environmentfilter
+def do_random(environment, seq):
+ """Return a random item from the sequence."""
+ try:
+ return choice(seq)
+ except IndexError:
+ return environment.undefined('No random item, sequence was empty.')
+
+
+def do_filesizeformat(value, binary=False):
+ """Format the value like a 'human-readable' file size (i.e. 13 KB,
+ 4.1 MB, 102 bytes, etc). Per default decimal prefixes are used (mega,
+ giga, etc.), if the second parameter is set to `True` the binary
+ prefixes are used (mebi, gibi).
+ """
+ bytes = float(value)
+ base = binary and 1024 or 1000
+ middle = binary and 'i' or ''
+ if bytes < base:
+ return "%d Byte%s" % (bytes, bytes != 1 and 's' or '')
+ elif bytes < base * base:
+ return "%.1f K%sB" % (bytes / base, middle)
+ elif bytes < base * base * base:
+ return "%.1f M%sB" % (bytes / (base * base), middle)
+ return "%.1f G%sB" % (bytes / (base * base * base), middle)
+
+
+def do_pprint(value, verbose=False):
+ """Pretty print a variable. Useful for debugging.
+
+ With Jinja 1.2 onwards you can pass it a parameter. If this parameter
+ is truthy the output will be more verbose (this requires `pretty`)
+ """
+ return pformat(value, verbose=verbose)
+
+
+@evalcontextfilter
+def do_urlize(eval_ctx, value, trim_url_limit=None, nofollow=False):
+ """Converts URLs in plain text into clickable links.
+
+ If you pass the filter an additional integer it will shorten the urls
+ to that number. Also a third argument exists that makes the urls
+ "nofollow":
+
+ .. sourcecode:: jinja
+
+ {{ mytext|urlize(40, true) }}
+ links are shortened to 40 chars and defined with rel="nofollow"
+ """
+ rv = urlize(value, trim_url_limit, nofollow)
+ if eval_ctx.autoescape:
+ rv = Markup(rv)
+ return rv
+
+
+def do_indent(s, width=4, indentfirst=False):
+ """Return a copy of the passed string, each line indented by
+ 4 spaces. The first line is not indented. If you want to
+ change the number of spaces or indent the first line too
+ you can pass additional parameters to the filter:
+
+ .. sourcecode:: jinja
+
+ {{ mytext|indent(2, true) }}
+ indent by two spaces and indent the first line too.
+ """
+ indention = u' ' * width
+ rv = (u'\n' + indention).join(s.splitlines())
+ if indentfirst:
+ rv = indention + rv
+ return rv
+
+
+def do_truncate(s, length=255, killwords=False, end='...'):
+ """Return a truncated copy of the string. The length is specified
+ with the first parameter which defaults to ``255``. If the second
+ parameter is ``true`` the filter will cut the text at length. Otherwise
+ it will try to save the last word. If the text was in fact
+ truncated it will append an ellipsis sign (``"..."``). If you want a
+ different ellipsis sign than ``"..."`` you can specify it using the
+ third parameter.
+
+ .. sourcecode jinja::
+
+ {{ mytext|truncate(300, false, '»') }}
+ truncate mytext to 300 chars, don't split up words, use a
+ right pointing double arrow as ellipsis sign.
+ """
+ if len(s) <= length:
+ return s
+ elif killwords:
+ return s[:length] + end
+ words = s.split(' ')
+ result = []
+ m = 0
+ for word in words:
+ m += len(word) + 1
+ if m > length:
+ break
+ result.append(word)
+ result.append(end)
+ return u' '.join(result)
+
+
+def do_wordwrap(s, width=79, break_long_words=True):
+ """
+ Return a copy of the string passed to the filter wrapped after
+ ``79`` characters. You can override this default using the first
+ parameter. If you set the second parameter to `false` Jinja will not
+ split words apart if they are longer than `width`.
+ """
+ import textwrap
+ return u'\n'.join(textwrap.wrap(s, width=width, expand_tabs=False,
+ replace_whitespace=False,
+ break_long_words=break_long_words))
+
+
+def do_wordcount(s):
+ """Count the words in that string."""
+ return len(_word_re.findall(s))
+
+
+def do_int(value, default=0):
+ """Convert the value into an integer. If the
+ conversion doesn't work it will return ``0``. You can
+ override this default using the first parameter.
+ """
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ # this quirk is necessary so that "42.23"|int gives 42.
+ try:
+ return int(float(value))
+ except (TypeError, ValueError):
+ return default
+
+
+def do_float(value, default=0.0):
+ """Convert the value into a floating point number. If the
+ conversion doesn't work it will return ``0.0``. You can
+ override this default using the first parameter.
+ """
+ try:
+ return float(value)
+ except (TypeError, ValueError):
+ return default
+
+
+def do_format(value, *args, **kwargs):
+ """
+ Apply python string formatting on an object:
+
+ .. sourcecode:: jinja
+
+ {{ "%s - %s"|format("Hello?", "Foo!") }}
+ -> Hello? - Foo!
+ """
+ if args and kwargs:
+ raise FilterArgumentError('can\'t handle positional and keyword '
+ 'arguments at the same time')
+ return soft_unicode(value) % (kwargs or args)
+
+
+def do_trim(value):
+ """Strip leading and trailing whitespace."""
+ return soft_unicode(value).strip()
+
+
+def do_striptags(value):
+ """Strip SGML/XML tags and replace adjacent whitespace by one space.
+ """
+ if hasattr(value, '__html__'):
+ value = value.__html__()
+ return Markup(unicode(value)).striptags()
+
+
+def do_slice(value, slices, fill_with=None):
+ """Slice an iterator and return a list of lists containing
+ those items. Useful if you want to create a div containing
+ three ul tags that represent columns:
+
+ .. sourcecode:: html+jinja
+
+ <div class="columwrapper">
+ {%- for column in items|slice(3) %}
+ <ul class="column-{{ loop.index }}">
+ {%- for item in column %}
+ <li>{{ item }}</li>
+ {%- endfor %}
+ </ul>
+ {%- endfor %}
+ </div>
+
+ If you pass it a second argument it's used to fill missing
+ values on the last iteration.
+ """
+ seq = list(value)
+ length = len(seq)
+ items_per_slice = length // slices
+ slices_with_extra = length % slices
+ offset = 0
+ for slice_number in xrange(slices):
+ start = offset + slice_number * items_per_slice
+ if slice_number < slices_with_extra:
+ offset += 1
+ end = offset + (slice_number + 1) * items_per_slice
+ tmp = seq[start:end]
+ if fill_with is not None and slice_number >= slices_with_extra:
+ tmp.append(fill_with)
+ yield tmp
+
+
+def do_batch(value, linecount, fill_with=None):
+ """
+ A filter that batches items. It works pretty much like `slice`
+ just the other way round. It returns a list of lists with the
+ given number of items. If you provide a second parameter this
+ is used to fill missing items. See this example:
+
+ .. sourcecode:: html+jinja
+
+ <table>
+ {%- for row in items|batch(3, ' ') %}
+ <tr>
+ {%- for column in row %}
+ <td>{{ column }}</td>
+ {%- endfor %}
+ </tr>
+ {%- endfor %}
+ </table>
+ """
+ result = []
+ tmp = []
+ for item in value:
+ if len(tmp) == linecount:
+ yield tmp
+ tmp = []
+ tmp.append(item)
+ if tmp:
+ if fill_with is not None and len(tmp) < linecount:
+ tmp += [fill_with] * (linecount - len(tmp))
+ yield tmp
+
+
+def do_round(value, precision=0, method='common'):
+ """Round the number to a given precision. The first
+ parameter specifies the precision (default is ``0``), the
+ second the rounding method:
+
+ - ``'common'`` rounds either up or down
+ - ``'ceil'`` always rounds up
+ - ``'floor'`` always rounds down
+
+ If you don't specify a method ``'common'`` is used.
+
+ .. sourcecode:: jinja
+
+ {{ 42.55|round }}
+ -> 43.0
+ {{ 42.55|round(1, 'floor') }}
+ -> 42.5
+
+ Note that even if rounded to 0 precision, a float is returned. If
+ you need a real integer, pipe it through `int`:
+
+ .. sourcecode:: jinja
+
+ {{ 42.55|round|int }}
+ -> 43
+ """
+ if not method in ('common', 'ceil', 'floor'):
+ raise FilterArgumentError('method must be common, ceil or floor')
+ if method == 'common':
+ return round(value, precision)
+ func = getattr(math, method)
+ return func(value * (10 ** precision)) / (10 ** precision)
+
+
+@environmentfilter
+def do_groupby(environment, value, attribute):
+ """Group a sequence of objects by a common attribute.
+
+ If you for example have a list of dicts or objects that represent persons
+ with `gender`, `first_name` and `last_name` attributes and you want to
+ group all users by genders you can do something like the following
+ snippet:
+
+ .. sourcecode:: html+jinja
+
+ <ul>
+ {% for group in persons|groupby('gender') %}
+ <li>{{ group.grouper }}<ul>
+ {% for person in group.list %}
+ <li>{{ person.first_name }} {{ person.last_name }}</li>
+ {% endfor %}</ul></li>
+ {% endfor %}
+ </ul>
+
+ Additionally it's possible to use tuple unpacking for the grouper and
+ list:
+
+ .. sourcecode:: html+jinja
+
+ <ul>
+ {% for grouper, list in persons|groupby('gender') %}
+ ...
+ {% endfor %}
+ </ul>
+
+ As you can see the item we're grouping by is stored in the `grouper`
+ attribute and the `list` contains all the objects that have this grouper
+ in common.
+ """
+ expr = lambda x: environment.getitem(x, attribute)
+ return sorted(map(_GroupTuple, groupby(sorted(value, key=expr), expr)))
+
+
+class _GroupTuple(tuple):
+ __slots__ = ()
+ grouper = property(itemgetter(0))
+ list = property(itemgetter(1))
+
+ def __new__(cls, (key, value)):
+ return tuple.__new__(cls, (key, list(value)))
+
+
+def do_list(value):
+ """Convert the value into a list. If it was a string the returned list
+ will be a list of characters.
+ """
+ return list(value)
+
+
+def do_mark_safe(value):
+ """Mark the value as safe which means that in an environment with automatic
+ escaping enabled this variable will not be escaped.
+ """
+ return Markup(value)
+
+
+def do_mark_unsafe(value):
+ """Mark a value as unsafe. This is the reverse operation for :func:`safe`."""
+ return unicode(value)
+
+
+def do_reverse(value):
+ """Reverse the object or return an iterator the iterates over it the other
+ way round.
+ """
+ if isinstance(value, basestring):
+ return value[::-1]
+ try:
+ return reversed(value)
+ except TypeError:
+ try:
+ rv = list(value)
+ rv.reverse()
+ return rv
+ except TypeError:
+ raise FilterArgumentError('argument must be iterable')
+
+
+@environmentfilter
+def do_attr(environment, obj, name):
+ """Get an attribute of an object. ``foo|attr("bar")`` works like
+ ``foo["bar"]`` just that always an attribute is returned and items are not
+ looked up.
+
+ See :ref:`Notes on subscriptions <notes-on-subscriptions>` for more details.
+ """
+ try:
+ name = str(name)
+ except UnicodeError:
+ pass
+ else:
+ try:
+ value = getattr(obj, name)
+ except AttributeError:
+ pass
+ else:
+ if environment.sandboxed and not \
+ environment.is_safe_attribute(obj, name, value):
+ return environment.unsafe_undefined(obj, name)
+ return value
+ return environment.undefined(obj=obj, name=name)
+
+
+FILTERS = {
+ 'attr': do_attr,
+ 'replace': do_replace,
+ 'upper': do_upper,
+ 'lower': do_lower,
+ 'escape': escape,
+ 'e': escape,
+ 'forceescape': do_forceescape,
+ 'capitalize': do_capitalize,
+ 'title': do_title,
+ 'default': do_default,
+ 'd': do_default,
+ 'join': do_join,
+ 'count': len,
+ 'dictsort': do_dictsort,
+ 'sort': do_sort,
+ 'length': len,
+ 'reverse': do_reverse,
+ 'center': do_center,
+ 'indent': do_indent,
+ 'title': do_title,
+ 'capitalize': do_capitalize,
+ 'first': do_first,
+ 'last': do_last,
+ 'random': do_random,
+ 'filesizeformat': do_filesizeformat,
+ 'pprint': do_pprint,
+ 'truncate': do_truncate,
+ 'wordwrap': do_wordwrap,
+ 'wordcount': do_wordcount,
+ 'int': do_int,
+ 'float': do_float,
+ 'string': soft_unicode,
+ 'list': do_list,
+ 'urlize': do_urlize,
+ 'format': do_format,
+ 'trim': do_trim,
+ 'striptags': do_striptags,
+ 'slice': do_slice,
+ 'batch': do_batch,
+ 'sum': sum,
+ 'abs': abs,
+ 'round': do_round,
+ 'groupby': do_groupby,
+ 'safe': do_mark_safe,
+ 'xmlattr': do_xmlattr
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c3ea59f/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py
new file mode 100644
index 0000000..0c385c1
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py
@@ -0,0 +1,681 @@
+# -*- coding: utf-8 -*-
+"""
+ ambari_jinja2.lexer
+ ~~~~~~~~~~~~
+
+ This module implements a Jinja / Python combination lexer. The
+ `Lexer` class provided by this module is used to do some preprocessing
+ for Jinja.
+
+ On the one hand it filters out invalid operators like the bitshift
+ operators we don't allow in templates. On the other hand it separates
+ template code and python code in expressions.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+from operator import itemgetter
+from collections import deque
+from ambari_jinja2.exceptions import TemplateSyntaxError
+from ambari_jinja2.utils import LRUCache, next
+
+
+# cache for the lexers. Exists in order to be able to have multiple
+# environments with the same lexer
+_lexer_cache = LRUCache(50)
+
+# static regular expressions
+whitespace_re = re.compile(r'\s+', re.U)
+string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
+ r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
+integer_re = re.compile(r'\d+')
+
+# we use the unicode identifier rule if this python version is able
+# to handle unicode identifiers, otherwise the standard ASCII one.
+try:
+ compile('föö', '<unknown>', 'eval')
+except SyntaxError:
+ name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b')
+else:
+ from ambari_jinja2 import _stringdefs
+ name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start,
+ _stringdefs.xid_continue))
+
+float_re = re.compile(r'(?<!\.)\d+\.\d+')
+newline_re = re.compile(r'(\r\n|\r|\n)')
+
+# internal the tokens and keep references to them
+TOKEN_ADD = intern('add')
+TOKEN_ASSIGN = intern('assign')
+TOKEN_COLON = intern('colon')
+TOKEN_COMMA = intern('comma')
+TOKEN_DIV = intern('div')
+TOKEN_DOT = intern('dot')
+TOKEN_EQ = intern('eq')
+TOKEN_FLOORDIV = intern('floordiv')
+TOKEN_GT = intern('gt')
+TOKEN_GTEQ = intern('gteq')
+TOKEN_LBRACE = intern('lbrace')
+TOKEN_LBRACKET = intern('lbracket')
+TOKEN_LPAREN = intern('lparen')
+TOKEN_LT = intern('lt')
+TOKEN_LTEQ = intern('lteq')
+TOKEN_MOD = intern('mod')
+TOKEN_MUL = intern('mul')
+TOKEN_NE = intern('ne')
+TOKEN_PIPE = intern('pipe')
+TOKEN_POW = intern('pow')
+TOKEN_RBRACE = intern('rbrace')
+TOKEN_RBRACKET = intern('rbracket')
+TOKEN_RPAREN = intern('rparen')
+TOKEN_SEMICOLON = intern('semicolon')
+TOKEN_SUB = intern('sub')
+TOKEN_TILDE = intern('tilde')
+TOKEN_WHITESPACE = intern('whitespace')
+TOKEN_FLOAT = intern('float')
+TOKEN_INTEGER = intern('integer')
+TOKEN_NAME = intern('name')
+TOKEN_STRING = intern('string')
+TOKEN_OPERATOR = intern('operator')
+TOKEN_BLOCK_BEGIN = intern('block_begin')
+TOKEN_BLOCK_END = intern('block_end')
+TOKEN_VARIABLE_BEGIN = intern('variable_begin')
+TOKEN_VARIABLE_END = intern('variable_end')
+TOKEN_RAW_BEGIN = intern('raw_begin')
+TOKEN_RAW_END = intern('raw_end')
+TOKEN_COMMENT_BEGIN = intern('comment_begin')
+TOKEN_COMMENT_END = intern('comment_end')
+TOKEN_COMMENT = intern('comment')
+TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin')
+TOKEN_LINESTATEMENT_END = intern('linestatement_end')
+TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin')
+TOKEN_LINECOMMENT_END = intern('linecomment_end')
+TOKEN_LINECOMMENT = intern('linecomment')
+TOKEN_DATA = intern('data')
+TOKEN_INITIAL = intern('initial')
+TOKEN_EOF = intern('eof')
+
+# bind operators to token types
+operators = {
+ '+': TOKEN_ADD,
+ '-': TOKEN_SUB,
+ '/': TOKEN_DIV,
+ '//': TOKEN_FLOORDIV,
+ '*': TOKEN_MUL,
+ '%': TOKEN_MOD,
+ '**': TOKEN_POW,
+ '~': TOKEN_TILDE,
+ '[': TOKEN_LBRACKET,
+ ']': TOKEN_RBRACKET,
+ '(': TOKEN_LPAREN,
+ ')': TOKEN_RPAREN,
+ '{': TOKEN_LBRACE,
+ '}': TOKEN_RBRACE,
+ '==': TOKEN_EQ,
+ '!=': TOKEN_NE,
+ '>': TOKEN_GT,
+ '>=': TOKEN_GTEQ,
+ '<': TOKEN_LT,
+ '<=': TOKEN_LTEQ,
+ '=': TOKEN_ASSIGN,
+ '.': TOKEN_DOT,
+ ':': TOKEN_COLON,
+ '|': TOKEN_PIPE,
+ ',': TOKEN_COMMA,
+ ';': TOKEN_SEMICOLON
+}
+
+reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
+assert len(operators) == len(reverse_operators), 'operators dropped'
+operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
+ sorted(operators, key=lambda x: -len(x))))
+
+ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
+ TOKEN_COMMENT_END, TOKEN_WHITESPACE,
+ TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN,
+ TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT])
+ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
+ TOKEN_COMMENT, TOKEN_LINECOMMENT])
+
+
+def _describe_token_type(token_type):
+ if token_type in reverse_operators:
+ return reverse_operators[token_type]
+ return {
+ TOKEN_COMMENT_BEGIN: 'begin of comment',
+ TOKEN_COMMENT_END: 'end of comment',
+ TOKEN_COMMENT: 'comment',
+ TOKEN_LINECOMMENT: 'comment',
+ TOKEN_BLOCK_BEGIN: 'begin of statement block',
+ TOKEN_BLOCK_END: 'end of statement block',
+ TOKEN_VARIABLE_BEGIN: 'begin of print statement',
+ TOKEN_VARIABLE_END: 'end of print statement',
+ TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement',
+ TOKEN_LINESTATEMENT_END: 'end of line statement',
+ TOKEN_DATA: 'template data / text',
+ TOKEN_EOF: 'end of template'
+ }.get(token_type, token_type)
+
+
+def describe_token(token):
+ """Returns a description of the token."""
+ if token.type == 'name':
+ return token.value
+ return _describe_token_type(token.type)
+
+
+def describe_token_expr(expr):
+ """Like `describe_token` but for token expressions."""
+ if ':' in expr:
+ type, value = expr.split(':', 1)
+ if type == 'name':
+ return value
+ else:
+ type = expr
+ return _describe_token_type(type)
+
+
+def count_newlines(value):
+ """Count the number of newline characters in the string. This is
+ useful for extensions that filter a stream.
+ """
+ return len(newline_re.findall(value))
+
+
+def compile_rules(environment):
+ """Compiles all the rules from the environment into a list of rules."""
+ e = re.escape
+ rules = [
+ (len(environment.comment_start_string), 'comment',
+ e(environment.comment_start_string)),
+ (len(environment.block_start_string), 'block',
+ e(environment.block_start_string)),
+ (len(environment.variable_start_string), 'variable',
+ e(environment.variable_start_string))
+ ]
+
+ if environment.line_statement_prefix is not None:
+ rules.append((len(environment.line_statement_prefix), 'linestatement',
+ r'^\s*' + e(environment.line_statement_prefix)))
+ if environment.line_comment_prefix is not None:
+ rules.append((len(environment.line_comment_prefix), 'linecomment',
+ r'(?:^|(?<=\S))[^\S\r\n]*' +
+ e(environment.line_comment_prefix)))
+
+ return [x[1:] for x in sorted(rules, reverse=True)]
+
+
+class Failure(object):
+ """Class that raises a `TemplateSyntaxError` if called.
+ Used by the `Lexer` to specify known errors.
+ """
+
+ def __init__(self, message, cls=TemplateSyntaxError):
+ self.message = message
+ self.error_class = cls
+
+ def __call__(self, lineno, filename):
+ raise self.error_class(self.message, lineno, filename)
+
+
+class Token(tuple):
+ """Token class."""
+ __slots__ = ()
+ lineno, type, value = (property(itemgetter(x)) for x in range(3))
+
+ def __new__(cls, lineno, type, value):
+ return tuple.__new__(cls, (lineno, intern(str(type)), value))
+
+ def __str__(self):
+ if self.type in reverse_operators:
+ return reverse_operators[self.type]
+ elif self.type == 'name':
+ return self.value
+ return self.type
+
+ def test(self, expr):
+ """Test a token against a token expression. This can either be a
+ token type or ``'token_type:token_value'``. This can only test
+ against string values and types.
+ """
+ # here we do a regular string equality check as test_any is usually
+ # passed an iterable of not interned strings.
+ if self.type == expr:
+ return True
+ elif ':' in expr:
+ return expr.split(':', 1) == [self.type, self.value]
+ return False
+
+ def test_any(self, *iterable):
+ """Test against multiple token expressions."""
+ for expr in iterable:
+ if self.test(expr):
+ return True
+ return False
+
+ def __repr__(self):
+ return 'Token(%r, %r, %r)' % (
+ self.lineno,
+ self.type,
+ self.value
+ )
+
+
+class TokenStreamIterator(object):
+ """The iterator for tokenstreams. Iterate over the stream
+ until the eof token is reached.
+ """
+
+ def __init__(self, stream):
+ self.stream = stream
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ token = self.stream.current
+ if token.type is TOKEN_EOF:
+ self.stream.close()
+ raise StopIteration()
+ next(self.stream)
+ return token
+
+
+class TokenStream(object):
+ """A token stream is an iterable that yields :class:`Token`\s. The
+ parser however does not iterate over it but calls :meth:`next` to go
+ one token ahead. The current active token is stored as :attr:`current`.
+ """
+
+ def __init__(self, generator, name, filename):
+ self._next = iter(generator).next
+ self._pushed = deque()
+ self.name = name
+ self.filename = filename
+ self.closed = False
+ self.current = Token(1, TOKEN_INITIAL, '')
+ next(self)
+
+ def __iter__(self):
+ return TokenStreamIterator(self)
+
+ def __nonzero__(self):
+ return bool(self._pushed) or self.current.type is not TOKEN_EOF
+
+ eos = property(lambda x: not x, doc="Are we at the end of the stream?")
+
+ def push(self, token):
+ """Push a token back to the stream."""
+ self._pushed.append(token)
+
+ def look(self):
+ """Look at the next token."""
+ old_token = next(self)
+ result = self.current
+ self.push(result)
+ self.current = old_token
+ return result
+
+ def skip(self, n=1):
+ """Got n tokens ahead."""
+ for x in xrange(n):
+ next(self)
+
+ def next_if(self, expr):
+ """Perform the token test and return the token if it matched.
+ Otherwise the return value is `None`.
+ """
+ if self.current.test(expr):
+ return next(self)
+
+ def skip_if(self, expr):
+ """Like :meth:`next_if` but only returns `True` or `False`."""
+ return self.next_if(expr) is not None
+
+ def next(self):
+ """Go one token ahead and return the old one"""
+ rv = self.current
+ if self._pushed:
+ self.current = self._pushed.popleft()
+ elif self.current.type is not TOKEN_EOF:
+ try:
+ self.current = self._next()
+ except StopIteration:
+ self.close()
+ return rv
+
+ def close(self):
+ """Close the stream."""
+ self.current = Token(self.current.lineno, TOKEN_EOF, '')
+ self._next = None
+ self.closed = True
+
+ def expect(self, expr):
+ """Expect a given token type and return it. This accepts the same
+ argument as :meth:`ambari_jinja2.lexer.Token.test`.
+ """
+ if not self.current.test(expr):
+ expr = describe_token_expr(expr)
+ if self.current.type is TOKEN_EOF:
+ raise TemplateSyntaxError('unexpected end of template, '
+ 'expected %r.' % expr,
+ self.current.lineno,
+ self.name, self.filename)
+ raise TemplateSyntaxError("expected token %r, got %r" %
+ (expr, describe_token(self.current)),
+ self.current.lineno,
+ self.name, self.filename)
+ try:
+ return self.current
+ finally:
+ next(self)
+
+
+def get_lexer(environment):
+ """Return a lexer which is probably cached."""
+ key = (environment.block_start_string,
+ environment.block_end_string,
+ environment.variable_start_string,
+ environment.variable_end_string,
+ environment.comment_start_string,
+ environment.comment_end_string,
+ environment.line_statement_prefix,
+ environment.line_comment_prefix,
+ environment.trim_blocks,
+ environment.newline_sequence)
+ lexer = _lexer_cache.get(key)
+ if lexer is None:
+ lexer = Lexer(environment)
+ _lexer_cache[key] = lexer
+ return lexer
+
+
+class Lexer(object):
+ """Class that implements a lexer for a given environment. Automatically
+ created by the environment class, usually you don't have to do that.
+
+ Note that the lexer is not automatically bound to an environment.
+ Multiple environments can share the same lexer.
+ """
+
+ def __init__(self, environment):
+ # shortcuts
+ c = lambda x: re.compile(x, re.M | re.S)
+ e = re.escape
+
+ # lexing rules for tags
+ tag_rules = [
+ (whitespace_re, TOKEN_WHITESPACE, None),
+ (float_re, TOKEN_FLOAT, None),
+ (integer_re, TOKEN_INTEGER, None),
+ (name_re, TOKEN_NAME, None),
+ (string_re, TOKEN_STRING, None),
+ (operator_re, TOKEN_OPERATOR, None)
+ ]
+
+ # assamble the root lexing rule. because "|" is ungreedy
+ # we have to sort by length so that the lexer continues working
+ # as expected when we have parsing rules like <% for block and
+ # <%= for variables. (if someone wants asp like syntax)
+ # variables are just part of the rules if variable processing
+ # is required.
+ root_tag_rules = compile_rules(environment)
+
+ # block suffix if trimming is enabled
+ block_suffix_re = environment.trim_blocks and '\\n?' or ''
+
+ self.newline_sequence = environment.newline_sequence
+
+ # global lexing rules
+ self.rules = {
+ 'root': [
+ # directives
+ (c('(.*?)(?:%s)' % '|'.join(
+ [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+ e(environment.block_start_string),
+ e(environment.block_start_string),
+ e(environment.block_end_string),
+ e(environment.block_end_string)
+ )] + [
+ r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
+ for n, r in root_tag_rules
+ ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
+ # data
+ (c('.+'), TOKEN_DATA, None)
+ ],
+ # comments
+ TOKEN_COMMENT_BEGIN: [
+ (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
+ e(environment.comment_end_string),
+ e(environment.comment_end_string),
+ block_suffix_re
+ )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
+ (c('(.)'), (Failure('Missing end of comment tag'),), None)
+ ],
+ # blocks
+ TOKEN_BLOCK_BEGIN: [
+ (c('(?:\-%s\s*|%s)%s' % (
+ e(environment.block_end_string),
+ e(environment.block_end_string),
+ block_suffix_re
+ )), TOKEN_BLOCK_END, '#pop'),
+ ] + tag_rules,
+ # variables
+ TOKEN_VARIABLE_BEGIN: [
+ (c('\-%s\s*|%s' % (
+ e(environment.variable_end_string),
+ e(environment.variable_end_string)
+ )), TOKEN_VARIABLE_END, '#pop')
+ ] + tag_rules,
+ # raw block
+ TOKEN_RAW_BEGIN: [
+ (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
+ e(environment.block_start_string),
+ e(environment.block_start_string),
+ e(environment.block_end_string),
+ e(environment.block_end_string),
+ block_suffix_re
+ )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
+ (c('(.)'), (Failure('Missing end of raw directive'),), None)
+ ],
+ # line statements
+ TOKEN_LINESTATEMENT_BEGIN: [
+ (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
+ ] + tag_rules,
+ # line comments
+ TOKEN_LINECOMMENT_BEGIN: [
+ (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT,
+ TOKEN_LINECOMMENT_END), '#pop')
+ ]
+ }
+
+ def _normalize_newlines(self, value):
+ """Called for strings and template data to normlize it to unicode."""
+ return newline_re.sub(self.newline_sequence, value)
+
+ def tokenize(self, source, name=None, filename=None, state=None):
+ """Calls tokeniter + tokenize and wraps it in a token stream.
+ """
+ stream = self.tokeniter(source, name, filename, state)
+ return TokenStream(self.wrap(stream, name, filename), name, filename)
+
+ def wrap(self, stream, name=None, filename=None):
+ """This is called with the stream as returned by `tokenize` and wraps
+ every token in a :class:`Token` and converts the value.
+ """
+ for lineno, token, value in stream:
+ if token in ignored_tokens:
+ continue
+ elif token == 'linestatement_begin':
+ token = 'block_begin'
+ elif token == 'linestatement_end':
+ token = 'block_end'
+ # we are not interested in those tokens in the parser
+ elif token in ('raw_begin', 'raw_end'):
+ continue
+ elif token == 'data':
+ value = self._normalize_newlines(value)
+ elif token == 'keyword':
+ token = value
+ elif token == 'name':
+ value = str(value)
+ elif token == 'string':
+ # try to unescape string
+ try:
+ value = self._normalize_newlines(value[1:-1]) \
+ .encode('ascii', 'backslashreplace') \
+ .decode('unicode-escape')
+ except Exception, e:
+ msg = str(e).split(':')[-1].strip()
+ raise TemplateSyntaxError(msg, lineno, name, filename)
+ # if we can express it as bytestring (ascii only)
+ # we do that for support of semi broken APIs
+ # as datetime.datetime.strftime. On python 3 this
+ # call becomes a noop thanks to 2to3
+ try:
+ value = str(value)
+ except UnicodeError:
+ pass
+ elif token == 'integer':
+ value = int(value)
+ elif token == 'float':
+ value = float(value)
+ elif token == 'operator':
+ token = operators[value]
+ yield Token(lineno, token, value)
+
+ def tokeniter(self, source, name, filename=None, state=None):
+ """This method tokenizes the text and returns the tokens in a
+ generator. Use this method if you just want to tokenize a template.
+ """
+ source = '\n'.join(unicode(source).splitlines())
+ pos = 0
+ lineno = 1
+ stack = ['root']
+ if state is not None and state != 'root':
+ assert state in ('variable', 'block'), 'invalid state'
+ stack.append(state + '_begin')
+ else:
+ state = 'root'
+ statetokens = self.rules[stack[-1]]
+ source_length = len(source)
+
+ balancing_stack = []
+
+ while 1:
+ # tokenizer loop
+ for regex, tokens, new_state in statetokens:
+ m = regex.match(source, pos)
+ # if no match we try again with the next rule
+ if m is None:
+ continue
+
+ # we only match blocks and variables if brances / parentheses
+ # are balanced. continue parsing with the lower rule which
+ # is the operator rule. do this only if the end tags look
+ # like operators
+ if balancing_stack and \
+ tokens in ('variable_end', 'block_end',
+ 'linestatement_end'):
+ continue
+
+ # tuples support more options
+ if isinstance(tokens, tuple):
+ for idx, token in enumerate(tokens):
+ # failure group
+ if token.__class__ is Failure:
+ raise token(lineno, filename)
+ # bygroup is a bit more complex, in that case we
+ # yield for the current token the first named
+ # group that matched
+ elif token == '#bygroup':
+ for key, value in m.groupdict().iteritems():
+ if value is not None:
+ yield lineno, key, value
+ lineno += value.count('\n')
+ break
+ else:
+ raise RuntimeError('%r wanted to resolve '
+ 'the token dynamically'
+ ' but no group matched'
+ % regex)
+ # normal group
+ else:
+ data = m.group(idx + 1)
+ if data or token not in ignore_if_empty:
+ yield lineno, token, data
+ lineno += data.count('\n')
+
+ # strings as token just are yielded as it.
+ else:
+ data = m.group()
+ # update brace/parentheses balance
+ if tokens == 'operator':
+ if data == '{':
+ balancing_stack.append('}')
+ elif data == '(':
+ balancing_stack.append(')')
+ elif data == '[':
+ balancing_stack.append(']')
+ elif data in ('}', ')', ']'):
+ if not balancing_stack:
+ raise TemplateSyntaxError('unexpected \'%s\'' %
+ data, lineno, name,
+ filename)
+ expected_op = balancing_stack.pop()
+ if expected_op != data:
+ raise TemplateSyntaxError('unexpected \'%s\', '
+ 'expected \'%s\'' %
+ (data, expected_op),
+ lineno, name,
+ filename)
+ # yield items
+ if data or tokens not in ignore_if_empty:
+ yield lineno, tokens, data
+ lineno += data.count('\n')
+
+ # fetch new position into new variable so that we can check
+ # if there is a internal parsing error which would result
+ # in an infinite loop
+ pos2 = m.end()
+
+ # handle state changes
+ if new_state is not None:
+ # remove the uppermost state
+ if new_state == '#pop':
+ stack.pop()
+ # resolve the new state by group checking
+ elif new_state == '#bygroup':
+ for key, value in m.groupdict().iteritems():
+ if value is not None:
+ stack.append(key)
+ break
+ else:
+ raise RuntimeError('%r wanted to resolve the '
+ 'new state dynamically but'
+ ' no group matched' %
+ regex)
+ # direct state name given
+ else:
+ stack.append(new_state)
+ statetokens = self.rules[stack[-1]]
+ # we are still at the same position and no stack change.
+ # this means a loop without break condition, avoid that and
+ # raise error
+ elif pos2 == pos:
+ raise RuntimeError('%r yielded empty string without '
+ 'stack change' % regex)
+ # publish new function and start again
+ pos = pos2
+ break
+ # if loop terminated without break we havn't found a single match
+ # either we are at the end of the file or we have a problem
+ else:
+ # end of text
+ if pos >= source_length:
+ return
+ # something went wrong
+ raise TemplateSyntaxError('unexpected char %r at %d' %
+ (source[pos], pos), lineno,
+ name, filename)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c3ea59f/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py
new file mode 100644
index 0000000..b25be08
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py
@@ -0,0 +1,449 @@
+# -*- coding: utf-8 -*-
+"""
+ ambari_jinja2.loaders
+ ~~~~~~~~~~~~~~
+
+ Jinja loader classes.
+
+ :copyright: (c) 2010 by the Jinja Team.
+ :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import weakref
+from types import ModuleType
+from os import path
+try:
+ from hashlib import sha1
+except ImportError:
+ from sha import new as sha1
+from ambari_jinja2.exceptions import TemplateNotFound
+from ambari_jinja2.utils import LRUCache, open_if_exists, internalcode
+
+
+def split_template_path(template):
+ """Split a path into segments and perform a sanity check. If it detects
+ '..' in the path it will raise a `TemplateNotFound` error.
+ """
+ pieces = []
+ for piece in template.split('/'):
+ if path.sep in piece \
+ or (path.altsep and path.altsep in piece) or \
+ piece == path.pardir:
+ raise TemplateNotFound(template)
+ elif piece and piece != '.':
+ pieces.append(piece)
+ return pieces
+
+
+class BaseLoader(object):
+ """Baseclass for all loaders. Subclass this and override `get_source` to
+ implement a custom loading mechanism. The environment provides a
+ `get_template` method that calls the loader's `load` method to get the
+ :class:`Template` object.
+
+ A very basic example for a loader that looks up templates on the file
+ system could look like this::
+
+ from ambari_jinja2 import BaseLoader, TemplateNotFound
+ from os.path import join, exists, getmtime
+
+ class MyLoader(BaseLoader):
+
+ def __init__(self, path):
+ self.path = path
+
+ def get_source(self, environment, template):
+ path = join(self.path, template)
+ if not exists(path):
+ raise TemplateNotFound(template)
+ mtime = getmtime(path)
+ with file(path) as f:
+ source = f.read().decode('utf-8')
+ return source, path, lambda: mtime == getmtime(path)
+ """
+
+ #: if set to `False` it indicates that the loader cannot provide access
+ #: to the source of templates.
+ #:
+ #: .. versionadded:: 2.4
+ has_source_access = True
+
+ def get_source(self, environment, template):
+ """Get the template source, filename and reload helper for a template.
+ It's passed the environment and template name and has to return a
+ tuple in the form ``(source, filename, uptodate)`` or raise a
+ `TemplateNotFound` error if it can't locate the template.
+
+ The source part of the returned tuple must be the source of the
+ template as unicode string or a ASCII bytestring. The filename should
+ be the name of the file on the filesystem if it was loaded from there,
+ otherwise `None`. The filename is used by python for the tracebacks
+ if no loader extension is used.
+
+ The last item in the tuple is the `uptodate` function. If auto
+ reloading is enabled it's always called to check if the template
+ changed. No arguments are passed so the function must store the
+ old state somewhere (for example in a closure). If it returns `False`
+ the template will be reloaded.
+ """
+ if not self.has_source_access:
+ raise RuntimeError('%s cannot provide access to the source' %
+ self.__class__.__name__)
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ """Iterates over all templates. If the loader does not support that
+ it should raise a :exc:`TypeError` which is the default behavior.
+ """
+ raise TypeError('this loader cannot iterate over all templates')
+
+ @internalcode
+ def load(self, environment, name, globals=None):
+ """Loads a template. This method looks up the template in the cache
+ or loads one by calling :meth:`get_source`. Subclasses should not
+ override this method as loaders working on collections of other
+ loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
+ will not call this method but `get_source` directly.
+ """
+ code = None
+ if globals is None:
+ globals = {}
+
+ # first we try to get the source for this template together
+ # with the filename and the uptodate function.
+ source, filename, uptodate = self.get_source(environment, name)
+
+ # try to load the code from the bytecode cache if there is a
+ # bytecode cache configured.
+ bcc = environment.bytecode_cache
+ if bcc is not None:
+ bucket = bcc.get_bucket(environment, name, filename, source)
+ code = bucket.code
+
+ # if we don't have code so far (not cached, no longer up to
+ # date) etc. we compile the template
+ if code is None:
+ code = environment.compile(source, name, filename)
+
+ # if the bytecode cache is available and the bucket doesn't
+ # have a code so far, we give the bucket the new code and put
+ # it back to the bytecode cache.
+ if bcc is not None and bucket.code is None:
+ bucket.code = code
+ bcc.set_bucket(bucket)
+
+ return environment.template_class.from_code(environment, code,
+ globals, uptodate)
+
+
+class FileSystemLoader(BaseLoader):
+ """Loads templates from the file system. This loader can find templates
+ in folders on the file system and is the preferred way to load them.
+
+ The loader takes the path to the templates as string, or if multiple
+ locations are wanted a list of them which is then looked up in the
+ given order:
+
+ >>> loader = FileSystemLoader('/path/to/templates')
+ >>> loader = FileSystemLoader(['/path/to/templates', '/other/path'])
+
+ Per default the template encoding is ``'utf-8'`` which can be changed
+ by setting the `encoding` parameter to something else.
+ """
+
+ def __init__(self, searchpath, encoding='utf-8'):
+ if isinstance(searchpath, basestring):
+ searchpath = [searchpath]
+ self.searchpath = list(searchpath)
+ self.encoding = encoding
+
+ def get_source(self, environment, template):
+ pieces = split_template_path(template)
+ for searchpath in self.searchpath:
+ filename = path.join(searchpath, *pieces)
+ f = open_if_exists(filename)
+ if f is None:
+ continue
+ try:
+ contents = f.read().decode(self.encoding)
+ finally:
+ f.close()
+
+ mtime = path.getmtime(filename)
+ def uptodate():
+ try:
+ return path.getmtime(filename) == mtime
+ except OSError:
+ return False
+ return contents, filename, uptodate
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ found = set()
+ for searchpath in self.searchpath:
+ for dirpath, dirnames, filenames in os.walk(searchpath):
+ for filename in filenames:
+ template = os.path.join(dirpath, filename) \
+ [len(searchpath):].strip(os.path.sep) \
+ .replace(os.path.sep, '/')
+ if template[:2] == './':
+ template = template[2:]
+ if template not in found:
+ found.add(template)
+ return sorted(found)
+
+
+class PackageLoader(BaseLoader):
+ """Load templates from python eggs or packages. It is constructed with
+ the name of the python package and the path to the templates in that
+ package::
+
+ loader = PackageLoader('mypackage', 'views')
+
+ If the package path is not given, ``'templates'`` is assumed.
+
+ Per default the template encoding is ``'utf-8'`` which can be changed
+ by setting the `encoding` parameter to something else. Due to the nature
+ of eggs it's only possible to reload templates if the package was loaded
+ from the file system and not a zip file.
+ """
+
+ def __init__(self, package_name, package_path='templates',
+ encoding='utf-8'):
+ from pkg_resources import DefaultProvider, ResourceManager, \
+ get_provider
+ provider = get_provider(package_name)
+ self.encoding = encoding
+ self.manager = ResourceManager()
+ self.filesystem_bound = isinstance(provider, DefaultProvider)
+ self.provider = provider
+ self.package_path = package_path
+
+ def get_source(self, environment, template):
+ pieces = split_template_path(template)
+ p = '/'.join((self.package_path,) + tuple(pieces))
+ if not self.provider.has_resource(p):
+ raise TemplateNotFound(template)
+
+ filename = uptodate = None
+ if self.filesystem_bound:
+ filename = self.provider.get_resource_filename(self.manager, p)
+ mtime = path.getmtime(filename)
+ def uptodate():
+ try:
+ return path.getmtime(filename) == mtime
+ except OSError:
+ return False
+
+ source = self.provider.get_resource_string(self.manager, p)
+ return source.decode(self.encoding), filename, uptodate
+
+ def list_templates(self):
+ path = self.package_path
+ if path[:2] == './':
+ path = path[2:]
+ elif path == '.':
+ path = ''
+ offset = len(path)
+ results = []
+ def _walk(path):
+ for filename in self.provider.resource_listdir(path):
+ fullname = path + '/' + filename
+ if self.provider.resource_isdir(fullname):
+ for item in _walk(fullname):
+ results.append(item)
+ else:
+ results.append(fullname[offset:].lstrip('/'))
+ _walk(path)
+ results.sort()
+ return results
+
+
+class DictLoader(BaseLoader):
+ """Loads a template from a python dict. It's passed a dict of unicode
+ strings bound to template names. This loader is useful for unittesting:
+
+ >>> loader = DictLoader({'index.html': 'source here'})
+
+ Because auto reloading is rarely useful this is disabled per default.
+ """
+
+ def __init__(self, mapping):
+ self.mapping = mapping
+
+ def get_source(self, environment, template):
+ if template in self.mapping:
+ source = self.mapping[template]
+ return source, None, lambda: source != self.mapping.get(template)
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ return sorted(self.mapping)
+
+
+class FunctionLoader(BaseLoader):
+ """A loader that is passed a function which does the loading. The
+ function becomes the name of the template passed and has to return either
+ an unicode string with the template source, a tuple in the form ``(source,
+ filename, uptodatefunc)`` or `None` if the template does not exist.
+
+ >>> def load_template(name):
+ ... if name == 'index.html':
+ ... return '...'
+ ...
+ >>> loader = FunctionLoader(load_template)
+
+ The `uptodatefunc` is a function that is called if autoreload is enabled
+ and has to return `True` if the template is still up to date. For more
+ details have a look at :meth:`BaseLoader.get_source` which has the same
+ return value.
+ """
+
+ def __init__(self, load_func):
+ self.load_func = load_func
+
+ def get_source(self, environment, template):
+ rv = self.load_func(template)
+ if rv is None:
+ raise TemplateNotFound(template)
+ elif isinstance(rv, basestring):
+ return rv, None, None
+ return rv
+
+
+class PrefixLoader(BaseLoader):
+ """A loader that is passed a dict of loaders where each loader is bound
+ to a prefix. The prefix is delimited from the template by a slash per
+ default, which can be changed by setting the `delimiter` argument to
+ something else::
+
+ loader = PrefixLoader({
+ 'app1': PackageLoader('mypackage.app1'),
+ 'app2': PackageLoader('mypackage.app2')
+ })
+
+ By loading ``'app1/index.html'`` the file from the app1 package is loaded,
+ by loading ``'app2/index.html'`` the file from the second.
+ """
+
+ def __init__(self, mapping, delimiter='/'):
+ self.mapping = mapping
+ self.delimiter = delimiter
+
+ def get_source(self, environment, template):
+ try:
+ prefix, name = template.split(self.delimiter, 1)
+ loader = self.mapping[prefix]
+ except (ValueError, KeyError):
+ raise TemplateNotFound(template)
+ try:
+ return loader.get_source(environment, name)
+ except TemplateNotFound:
+ # re-raise the exception with the correct fileame here.
+ # (the one that includes the prefix)
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ result = []
+ for prefix, loader in self.mapping.iteritems():
+ for template in loader.list_templates():
+ result.append(prefix + self.delimiter + template)
+ return result
+
+
+class ChoiceLoader(BaseLoader):
+ """This loader works like the `PrefixLoader` just that no prefix is
+ specified. If a template could not be found by one loader the next one
+ is tried.
+
+ >>> loader = ChoiceLoader([
+ ... FileSystemLoader('/path/to/user/templates'),
+ ... FileSystemLoader('/path/to/system/templates')
+ ... ])
+
+ This is useful if you want to allow users to override builtin templates
+ from a different location.
+ """
+
+ def __init__(self, loaders):
+ self.loaders = loaders
+
+ def get_source(self, environment, template):
+ for loader in self.loaders:
+ try:
+ return loader.get_source(environment, template)
+ except TemplateNotFound:
+ pass
+ raise TemplateNotFound(template)
+
+ def list_templates(self):
+ found = set()
+ for loader in self.loaders:
+ found.update(loader.list_templates())
+ return sorted(found)
+
+
+class _TemplateModule(ModuleType):
+ """Like a normal module but with support for weak references"""
+
+
+class ModuleLoader(BaseLoader):
+ """This loader loads templates from precompiled templates.
+
+ Example usage:
+
+ >>> loader = ChoiceLoader([
+ ... ModuleLoader('/path/to/compiled/templates'),
+ ... FileSystemLoader('/path/to/templates')
+ ... ])
+ """
+
+ has_source_access = False
+
+ def __init__(self, path):
+ package_name = '_ambari_jinja2_module_templates_%x' % id(self)
+
+ # create a fake module that looks for the templates in the
+ # path given.
+ mod = _TemplateModule(package_name)
+ if isinstance(path, basestring):
+ path = [path]
+ else:
+ path = list(path)
+ mod.__path__ = path
+
+ sys.modules[package_name] = weakref.proxy(mod,
+ lambda x: sys.modules.pop(package_name, None))
+
+ # the only strong reference, the sys.modules entry is weak
+ # so that the garbage collector can remove it once the
+ # loader that created it goes out of business.
+ self.module = mod
+ self.package_name = package_name
+
+ @staticmethod
+ def get_template_key(name):
+ return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest()
+
+ @staticmethod
+ def get_module_filename(name):
+ return ModuleLoader.get_template_key(name) + '.py'
+
+ @internalcode
+ def load(self, environment, name, globals=None):
+ key = self.get_template_key(name)
+ module = '%s.%s' % (self.package_name, key)
+ mod = getattr(self.module, module, None)
+ if mod is None:
+ try:
+ mod = __import__(module, None, None, ['root'])
+ except ImportError:
+ raise TemplateNotFound(name)
+
+ # remove the entry from sys.modules, we only want the attribute
+ # on the module object we have stored on the loader.
+ sys.modules.pop(module, None)
+
+ return environment.template_class.from_module_dict(
+ environment, mod.__dict__, globals)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c3ea59f/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py
new file mode 100644
index 0000000..afe7771
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+ ambari_jinja2.meta
+ ~~~~~~~~~~~
+
+ This module implements various functions that exposes information about
+ templates that might be interesting for various kinds of applications.
+
+ :copyright: (c) 2010 by the Jinja Team, see AUTHORS for more details.
+ :license: BSD, see LICENSE for more details.
+"""
+from ambari_jinja2 import nodes
+from ambari_jinja2.compiler import CodeGenerator
+
+
+class TrackingCodeGenerator(CodeGenerator):
+ """We abuse the code generator for introspection."""
+
+ def __init__(self, environment):
+ CodeGenerator.__init__(self, environment, '<introspection>',
+ '<introspection>')
+ self.undeclared_identifiers = set()
+
+ def write(self, x):
+ """Don't write."""
+
+ def pull_locals(self, frame):
+ """Remember all undeclared identifiers."""
+ self.undeclared_identifiers.update(frame.identifiers.undeclared)
+
+
+def find_undeclared_variables(ast):
+ """Returns a set of all variables in the AST that will be looked up from
+ the context at runtime. Because at compile time it's not known which
+ variables will be used depending on the path the execution takes at
+ runtime, all variables are returned.
+
+ >>> from ambari_jinja2 import Environment, meta
+ >>> env = Environment()
+ >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
+ >>> meta.find_undeclared_variables(ast)
+ set(['bar'])
+
+ .. admonition:: Implementation
+
+ Internally the code generator is used for finding undeclared variables.
+ This is good to know because the code generator might raise a
+ :exc:`TemplateAssertionError` during compilation and as a matter of
+ fact this function can currently raise that exception as well.
+ """
+ codegen = TrackingCodeGenerator(ast.environment)
+ codegen.visit(ast)
+ return codegen.undeclared_identifiers
+
+
+def find_referenced_templates(ast):
+ """Finds all the referenced templates from the AST. This will return an
+ iterator over all the hardcoded template extensions, inclusions and
+ imports. If dynamic inheritance or inclusion is used, `None` will be
+ yielded.
+
+ >>> from ambari_jinja2 import Environment, meta
+ >>> env = Environment()
+ >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}')
+ >>> list(meta.find_referenced_templates(ast))
+ ['layout.html', None]
+
+ This function is useful for dependency tracking. For example if you want
+ to rebuild parts of the website after a layout template has changed.
+ """
+ for node in ast.find_all((nodes.Extends, nodes.FromImport, nodes.Import,
+ nodes.Include)):
+ if not isinstance(node.template, nodes.Const):
+ # a tuple with some non consts in there
+ if isinstance(node.template, (nodes.Tuple, nodes.List)):
+ for template_name in node.template.items:
+ # something const, only yield the strings and ignore
+ # non-string consts that really just make no sense
+ if isinstance(template_name, nodes.Const):
+ if isinstance(template_name.value, basestring):
+ yield template_name.value
+ # something dynamic in there
+ else:
+ yield None
+ # something dynamic we don't know about here
+ else:
+ yield None
+ continue
+ # constant is a basestring, direct template name
+ if isinstance(node.template.value, basestring):
+ yield node.template.value
+ # a tuple or list (latter *should* not happen) made of consts,
+ # yield the consts that are strings. We could warn here for
+ # non string values
+ elif isinstance(node, nodes.Include) and \
+ isinstance(node.template.value, (tuple, list)):
+ for template_name in node.template.value:
+ if isinstance(template_name, basestring):
+ yield template_name
+ # something else we don't care about, we could warn here
+ else:
+ yield None