utils/gyb.py

#!/usr/bin/env python3
# GYB: Generate Your Boilerplate (improved names welcome; at least
# this one's short).  See -h output for instructions

import io
import os
import re
import sys
import textwrap
import tokenize
from bisect import bisect
from io import StringIO


def get_line_starts(s):
    """Return a list containing the start index of each line in s.

    The list also contains a sentinel index for the end of the string,
    so there will be one more element in the list than there are lines
    in the string
    """
    starts = [0]

    for line in s.split('\n'):
        starts.append(starts[-1] + len(line) + 1)

    starts[-1] -= 1
    return starts


def strip_trailing_nl(s):
    """If s ends with a newline, drop it; else return s intact"""
    return s[:-1] if s.endswith('\n') else s


def split_lines(s):
    """Split s into a list of lines, each of which has a trailing newline

    If the lines are later concatenated, the result is s, possibly
    with a single appended newline.
    """
    return [line + '\n' for line in s.split('\n')]


# text on a line up to the first '$$', '${', or '%%'
literalText = r'(?: [^$\n%] | \$(?![${]) | %(?!%) )*'

# The part of an '%end' line that follows the '%' sign
linesClose = r'[\ \t]* end [\ \t]* (?: \# .* )? $'

# Note: Where "# Absorb" appears below, the regexp attempts to eat up
# through the end of ${...} and %{...}% constructs.  In reality we
# handle this with the Python tokenizer, which avoids mis-detections
# due to nesting, comments and strings.  This extra absorption in the
# regexp facilitates testing the regexp on its own, by preventing the
# interior of some of these constructs from being treated as literal
# text.
tokenize_re = re.compile(
    r'''
# %-lines and %{...}-blocks
    # \n? # absorb one preceding newline
    ^
    (?:
      (?P<gybLines>
        (?P<_indent> [\ \t]* % (?! [{%] ) [\ \t]* ) (?! [\ \t] | ''' +
    linesClose + r''' ) .*
        ( \n (?P=_indent) (?! ''' + linesClose + r''' ) .* ) *
      )
      | (?P<gybLinesClose> [\ \t]* % [ \t]* ''' + linesClose + r''' )
      | [\ \t]* (?P<gybBlockOpen> %\{  )
        (?: [^}]| \} (?!%) )* \}%  # Absorb
    )
    \n? # absorb one trailing newline

# Substitutions
| (?P<substitutionOpen> \$\{ )
  [^}]* \} # Absorb

# %% and $$ are literal % and $ respectively
| (?P<symbol>[$%]) (?P=symbol)

# Literal text
| (?P<literal> ''' + literalText + r'''
    (?:
      # newline that doesn't precede space+%
      (?: \n (?! [\ \t]* %[^%] ) )
      ''' + literalText + r'''
    )*
    \n?
  )
''', re.VERBOSE | re.MULTILINE)

gyb_block_close = re.compile(r'\}%[ \t]*\n?')


def token_pos_to_index(token_pos, start, line_starts):
    """Translate a tokenize (line, column) pair into an absolute
    position in source text given the position where we started
    tokenizing and a list that maps lines onto their starting
    character indexes.
    """
    relative_token_line_plus1, token_col = token_pos

    # line number where we started tokenizing
    start_line_num = bisect(line_starts, start) - 1

    # line number of the token in the whole text
    abs_token_line = relative_token_line_plus1 - 1 + start_line_num

    # if found in the first line, adjust the end column to account
    # for the extra text
    if relative_token_line_plus1 == 1:
        token_col += start - line_starts[start_line_num]

    # Sometimes tokenizer errors report a line beyond the last one
    if abs_token_line >= len(line_starts):
        return line_starts[-1]

    return line_starts[abs_token_line] + token_col


def tokenize_python_to_unmatched_close_curly(source_text, start, line_starts):
    """Apply Python's tokenize to source_text starting at index start
    while matching open and close curly braces.  When an unmatched
    close curly brace is found, return its index.  If not found,
    return len(source_text).  If there's a tokenization error, return
    the position of the error.
    """
    stream = StringIO(source_text)
    stream.seek(start)
    nesting = 0

    try:
        for kind, text, token_start, token_end, line_text \
                in tokenize.generate_tokens(stream.readline):

            if text == '{':
                nesting += 1
            elif text == '}':
                nesting -= 1
                if nesting < 0:
                    return token_pos_to_index(token_start, start, line_starts)

    except tokenize.TokenError as error:
        (message, error_pos) = error.args
        return token_pos_to_index(error_pos, start, line_starts)

    return len(source_text)


def tokenize_template(template_text):
    r"""Given the text of a template, returns an iterator over
    (tokenType, token, match) tuples.

    **Note**: this is template syntax tokenization, not Python
    tokenization.

    When a non-literal token is matched, a client may call
    iter.send(pos) on the iterator to reset the position in
    template_text at which scanning will resume.

    This function provides a base level of tokenization which is
    then refined by ParseContext.token_generator.

    >>> from pprint import *
    >>> pprint(list((kind, text) for kind, text, _ in tokenize_template(
    ...   '%for x in range(10):\n%  print x\n%end\njuicebox')))
    [('gybLines', '%for x in range(10):\n%  print x'),
     ('gybLinesClose', '%end'),
     ('literal', 'juicebox')]

    >>> pprint(list((kind, text) for kind, text, _ in tokenize_template(
    ... '''Nothing
    ... % if x:
    ... %    for i in range(3):
    ... ${i}
    ... %    end
    ... % else:
    ... THIS SHOULD NOT APPEAR IN THE OUTPUT
    ... ''')))
    [('literal', 'Nothing\n'),
     ('gybLines', '% if x:\n%    for i in range(3):'),
     ('substitutionOpen', '${'),
     ('literal', '\n'),
     ('gybLinesClose', '%    end'),
     ('gybLines', '% else:'),
     ('literal', 'THIS SHOULD NOT APPEAR IN THE OUTPUT\n')]

    >>> for kind, text, _ in tokenize_template('''
    ... This is $some$ literal stuff containing a ${substitution}
    ... followed by a %{...} block:
    ...   %{
    ...   # Python code
    ...   }%
    ... and here $${are} some %-lines:
    ...   % x = 1
    ...   % y = 2
    ...   % if z == 3:
    ...   %    print '${hello}'
    ...   % end
    ...   %    for x in zz:
    ...   %        print x
    ...     % # different indentation
    ... % twice
    ... and some lines that literally start with a %% token
    ... %% first line
    ...   %% second line
    ... '''):
    ...     print((kind, text.strip().split('\n',1)[0]))
    ('literal', 'This is $some$ literal stuff containing a')
    ('substitutionOpen', '${')
    ('literal', 'followed by a %{...} block:')
    ('gybBlockOpen', '%{')
    ('literal', 'and here ${are} some %-lines:')
    ('gybLines', '% x = 1')
    ('gybLinesClose', '% end')
    ('gybLines', '%    for x in zz:')
    ('gybLines', '% # different indentation')
    ('gybLines', '% twice')
    ('literal', 'and some lines that literally start with a % token')
    """
    pos = 0
    end = len(template_text)

    saved_literal = []
    literal_first_match = None

    while pos < end:
        m = tokenize_re.match(template_text, pos, end)

        # pull out the one matched key (ignoring internal patterns starting
        # with _)
        ((kind, text), ) = (
            (kind, text) for (kind, text) in m.groupdict().items()
            if text is not None and kind[0] != '_')

        if kind in ('literal', 'symbol'):
            if len(saved_literal) == 0:
                literal_first_match = m
            # literals and symbols get batched together
            saved_literal.append(text)
            pos = None
        else:
            # found a non-literal.  First yield any literal we've accumulated
            if saved_literal != []:
                yield 'literal', ''.join(saved_literal), literal_first_match
                saved_literal = []

            # Then yield the thing we found.  If we get a reply, it's
            # the place to resume tokenizing
            pos = yield kind, text, m

        # If we were not sent a new position by our client, resume
        # tokenizing at the end of this match.
        if pos is None:
            pos = m.end(0)
        else:
            # Client is not yet ready to process next token
            yield

    if saved_literal != []:
        yield 'literal', ''.join(saved_literal), literal_first_match


def split_gyb_lines(source_lines):
    r"""Return a list of lines at which to split the incoming source

    These positions represent the beginnings of python line groups that
    will require a matching %end construct if they are to be closed.

    >>> src = split_lines('''\
    ... if x:
    ...     print x
    ... if y: # trailing comment
    ...     print z
    ...     if z: # another comment\
    ... ''')
    >>> s = split_gyb_lines(src)
    >>> len(s)
    2
    >>> src[s[0]]
    '    print z\n'
    >>> s[1] - len(src)
    0

    >>> src = split_lines('''\
    ... if x:
    ...     if y: print 1
    ...     if z:
    ...         print 2
    ...     pass\
    ... ''')
    >>> s = split_gyb_lines(src)
    >>> len(s)
    1
    >>> src[s[0]]
    '    if y: print 1\n'

    >>> src = split_lines('''\
    ... if x:
    ...     if y:
    ...         print 1
    ...         print 2
    ... ''')
    >>> s = split_gyb_lines(src)
    >>> len(s)
    2
    >>> src[s[0]]
    '    if y:\n'
    >>> src[s[1]]
    '        print 1\n'
    """
    last_token_text, last_token_kind = None, None
    unmatched_indents = []

    dedents = 0
    try:
        for token_kind, token_text, token_start, \
            (token_end_line, token_end_col), line_text \
            in tokenize.generate_tokens(lambda i=iter(source_lines):
                                        next(i)):

            if token_kind in (tokenize.COMMENT, tokenize.ENDMARKER):
                continue

            if token_text == '\n' and last_token_text == ':':
                unmatched_indents.append(token_end_line)

            # The tokenizer appends dedents at EOF; don't consider
            # those as matching indentations.  Instead just save them
            # up...
            if last_token_kind == tokenize.DEDENT:
                dedents += 1
            # And count them later, when we see something real.
            if token_kind != tokenize.DEDENT and dedents > 0:
                unmatched_indents = unmatched_indents[:-dedents]
                dedents = 0

            last_token_text, last_token_kind = token_text, token_kind

    except tokenize.TokenError:
        # Let the later compile() call report the error
        return []

    if last_token_text == ':':
        unmatched_indents.append(len(source_lines))

    return unmatched_indents


def code_starts_with_dedent_keyword(source_lines):
    r"""Return True iff the incoming Python source_lines begin with "else",
    "elif", "except", or "finally".

    Initial comments and whitespace are ignored.

    >>> code_starts_with_dedent_keyword(split_lines('if x in y: pass'))
    False
    >>> code_starts_with_dedent_keyword(split_lines('except ifSomethingElse:'))
    True
    >>> code_starts_with_dedent_keyword(
    ...     split_lines('\n# comment\nelse: # yes'))
    True
    """
    token_text = None
    for token_kind, token_text, _, _, _ \
            in tokenize.generate_tokens(lambda i=iter(source_lines): next(i)):

        if token_kind != tokenize.COMMENT and token_text.strip() != '':
            break

    return token_text in ('else', 'elif', 'except', 'finally')


class ParseContext(object):

    """State carried through a parse of a template"""

    filename = ''
    template = ''
    line_starts = []
    code_start_line = -1
    code_text = None
    tokens = None       # The rest of the tokens
    close_lines = False

    def __init__(self, filename, template=None):
        self.filename = os.path.abspath(filename)
        if sys.platform == 'win32':
            self.filename = '/'.join(self.filename.split(os.sep))
        if template is None:
            with io.open(os.path.normpath(filename), encoding='utf-8') as f:
                self.template = f.read()
        else:
            self.template = template
        self.line_starts = get_line_starts(self.template)
        self.tokens = self.token_generator(tokenize_template(self.template))
        self.next_token()

    def pos_to_line(self, pos):
        return bisect(self.line_starts, pos) - 1

    def token_generator(self, base_tokens):
        r"""Given an iterator over (kind, text, match) triples (see
        tokenize_template above), return a refined iterator over
        token_kinds.

        Among other adjustments to the elements found by base_tokens,
        this refined iterator tokenizes python code embedded in
        template text to help determine its true extent.  The
        expression "base_tokens.send(pos)" is used to reset the index at
        which base_tokens resumes scanning the underlying text.

        >>> ctx = ParseContext('dummy', '''
        ... %for x in y:
        ... %    print x
        ... % end
        ... literally
        ... ''')
        >>> while ctx.token_kind:
        ...     print((ctx.token_kind, ctx.code_text or ctx.token_text))
        ...     ignored = ctx.next_token()
        ('literal', '\n')
        ('gybLinesOpen', 'for x in y:\n')
        ('gybLines', '    print x\n')
        ('gybLinesClose', '% end')
        ('literal', 'literally\n')

        >>> ctx = ParseContext('dummy',
        ... '''Nothing
        ... % if x:
        ... %    for i in range(3):
        ... ${i}
        ... %    end
        ... % else:
        ... THIS SHOULD NOT APPEAR IN THE OUTPUT
        ... ''')
        >>> while ctx.token_kind:
        ...     print((ctx.token_kind, ctx.code_text or ctx.token_text))
        ...     ignored = ctx.next_token()
        ('literal', 'Nothing\n')
        ('gybLinesOpen', 'if x:\n')
        ('gybLinesOpen', '   for i in range(3):\n')
        ('substitutionOpen', 'i')
        ('literal', '\n')
        ('gybLinesClose', '%    end')
        ('gybLinesOpen', 'else:\n')
        ('literal', 'THIS SHOULD NOT APPEAR IN THE OUTPUT\n')

        >>> ctx = ParseContext('dummy',
        ... '''% for x in [1, 2, 3]:
        ... %   if x == 1:
        ... literal1
        ... %   elif x > 1:  # add output line here to fix bug
        ... %     if x == 2:
        ... literal2
        ... %     end
        ... %   end
        ... % end
        ... ''')
        >>> while ctx.token_kind:
        ...     print((ctx.token_kind, ctx.code_text or ctx.token_text))
        ...     ignored = ctx.next_token()
        ('gybLinesOpen', 'for x in [1, 2, 3]:\n')
        ('gybLinesOpen', '  if x == 1:\n')
        ('literal', 'literal1\n')
        ('gybLinesOpen', 'elif x > 1:  # add output line here to fix bug\n')
        ('gybLinesOpen', '  if x == 2:\n')
        ('literal', 'literal2\n')
        ('gybLinesClose', '%     end')
        ('gybLinesClose', '%   end')
        ('gybLinesClose', '% end')
        """
        for self.token_kind, self.token_text, self.token_match in base_tokens:
            kind = self.token_kind
            self.code_text = None

            # Do we need to close the current lines?
            self.close_lines = kind == 'gybLinesClose'

            # %{...}% and ${...} constructs
            if kind.endswith('Open'):

                # Tokenize text that follows as Python up to an unmatched '}'
                code_start = self.token_match.end(kind)
                self.code_start_line = self.pos_to_line(code_start)

                close_pos = tokenize_python_to_unmatched_close_curly(
                    self.template, code_start, self.line_starts)
                self.code_text = self.template[code_start:close_pos]
                yield kind

                if (kind == 'gybBlockOpen'):
                    # Absorb any '}% <optional-comment> \n'
                    m2 = gyb_block_close.match(self.template, close_pos)
                    if not m2:
                        raise ValueError("Invalid block closure")
                    next_pos = m2.end(0)
                else:
                    assert kind == 'substitutionOpen'
                    # skip past the closing '}'
                    next_pos = close_pos + 1

                # Resume tokenizing after the end of the code.
                base_tokens.send(next_pos)

            elif kind == 'gybLines':

                self.code_start_line = self.pos_to_line(
                    self.token_match.start('gybLines'))
                indentation = self.token_match.group('_indent')

                # Strip off the leading indentation and %-sign
                source_lines = re.split(
                    '^' + re.escape(indentation),
                    self.token_match.group('gybLines') + '\n',
                    flags=re.MULTILINE)[1:]

                if code_starts_with_dedent_keyword(source_lines):
                    self.close_lines = True

                last_split = 0
                for line in split_gyb_lines(source_lines):
                    self.token_kind = 'gybLinesOpen'
                    self.code_text = ''.join(source_lines[last_split:line])
                    yield self.token_kind
                    last_split = line
                    self.code_start_line += line - last_split
                    self.close_lines = False

                self.code_text = ''.join(source_lines[last_split:])
                if self.code_text:
                    self.token_kind = 'gybLines'
                    yield self.token_kind
            else:
                yield self.token_kind

    def next_token(self):
        """Move to the next token"""
        for kind in self.tokens:
            return self.token_kind

        self.token_kind = None


_default_line_directive = \
    '// ###sourceLocation(file: "%(file)s", line: %(line)d)'


class ExecutionContext(object):

    """State we pass around during execution of a template"""

    def __init__(self, line_directive=_default_line_directive,
                 **local_bindings):
        self.local_bindings = local_bindings
        self.line_directive = line_directive
        self.local_bindings['__context__'] = self
        self.result_text = []
        self.last_file_line = None

    def append_text(self, text, file, line):
        # see if we need to inject a line marker
        if self.line_directive:
            if (file, line) != self.last_file_line:
                # We can only insert the line directive at a line break
                if len(self.result_text) == 0 \
                   or self.result_text[-1].endswith('\n'):
                    substitutions = {'file': file, 'line': line + 1}
                    format_str = self.line_directive + '\n'
                    self.result_text.append(format_str % substitutions)
                # But if the new text contains any line breaks, we can create
                # one
                elif '\n' in text:
                    i = text.find('\n')
                    self.result_text.append(text[:i + 1])
                    # and try again
                    self.append_text(text[i + 1:], file, line + 1)
                    return

        self.result_text.append(text)
        self.last_file_line = (file, line + text.count('\n'))


class ASTNode(object):

    """Abstract base class for template AST nodes"""

    def __init__(self):
        raise NotImplementedError("ASTNode.__init__ is not implemented.")

    def execute(self, context):
        raise NotImplementedError("ASTNode.execute is not implemented.")

    def __str__(self, indent=''):
        raise NotImplementedError("ASTNode.__str__ is not implemented.")

    def format_children(self, indent):
        if not self.children:
            return ' []'

        return '\n'.join(
            ['', indent + '['] +
            [x.__str__(indent + 4 * ' ') for x in self.children] +
            [indent + ']'])


class Block(ASTNode):

    """A sequence of other AST nodes, to be executed in order"""

    children = []

    def __init__(self, context):
        self.children = []

        while context.token_kind and not context.close_lines:
            if context.token_kind == 'literal':
                node = Literal
            else:
                node = Code
            self.children.append(node(context))

    def execute(self, context):
        for x in self.children:
            x.execute(context)

    def __str__(self, indent=''):
        return indent + 'Block:' + self.format_children(indent)


class Literal(ASTNode):

    """An AST node that generates literal text"""

    def __init__(self, context):
        self.text = context.token_text
        start_position = context.token_match.start(context.token_kind)
        self.start_line_number = context.pos_to_line(start_position)
        self.filename = context.filename
        context.next_token()

    def execute(self, context):
        context.append_text(self.text, self.filename, self.start_line_number)

    def __str__(self, indent=''):
        return '\n'.join(
            [indent + x for x in ['Literal:'] +
             strip_trailing_nl(self.text).split('\n')])


class Code(ASTNode):

    """An AST node that is evaluated as Python"""

    code = None
    children = ()
    kind = None

    def __init__(self, context):

        source = ''
        source_line_count = 0

        def accumulate_code():
            s = source + (context.code_start_line - source_line_count) * '\n' \
                + textwrap.dedent(context.code_text)
            line_count = context.code_start_line + \
                context.code_text.count('\n')
            context.next_token()
            return s, line_count

        eval_exec = 'exec'
        if context.token_kind.startswith('substitution'):
            eval_exec = 'eval'
            source, source_line_count = accumulate_code()
            source = '(' + source.strip() + ')'

        else:
            while context.token_kind == 'gybLinesOpen':
                source, source_line_count = accumulate_code()
                source += '    __children__[%d].execute(__context__)\n' % len(
                    self.children)
                source_line_count += 1

                self.children += (Block(context),)

            if context.token_kind == 'gybLinesClose':
                context.next_token()

        if context.token_kind == 'gybLines':
            source, source_line_count = accumulate_code()

            # Only handle a substitution as part of this code block if
            # we don't already have some %-lines.
        elif context.token_kind == 'gybBlockOpen':

            # Opening ${...} and %{...}% constructs
            source, source_line_count = accumulate_code()

        self.filename = context.filename
        self.start_line_number = context.code_start_line
        self.code = compile(source, context.filename, eval_exec)
        self.source = source

    def execute(self, context):
        # Save __children__ from the local bindings
        save_children = context.local_bindings.get('__children__')
        # Execute the code with our __children__ in scope
        context.local_bindings['__children__'] = self.children
        context.local_bindings['__file__'] = self.filename
        result = eval(self.code, context.local_bindings)

        if context.local_bindings['__children__'] is not self.children:
            raise ValueError("The code is not allowed to mutate __children__")
        # Restore the bindings
        context.local_bindings['__children__'] = save_children

        # If we got a result, the code was an expression, so append
        # its value
        if result is not None \
                or (isinstance(result, str) and result != ''):
            from numbers import Number, Integral
            result_string = None
            if isinstance(result, Number) and not isinstance(result, Integral):
                result_string = repr(result)
            elif isinstance(result, Integral) or isinstance(result, list):
                result_string = str(result)
            else:
                result_string = result
            context.append_text(
                result_string, self.filename, self.start_line_number)

    def __str__(self, indent=''):
        source_lines = re.sub(r'^\n', '', strip_trailing_nl(
            self.source), flags=re.MULTILINE).split('\n')
        if len(source_lines) == 1:
            s = indent + 'Code: {' + source_lines[0] + '}'
        else:
            s = indent + 'Code:\n' + indent + '{\n' + '\n'.join(
                indent + 4 * ' ' + line for line in source_lines
            ) + '\n' + indent + '}'
        return s + self.format_children(indent)


def expand(filename, line_directive=_default_line_directive, **local_bindings):
    r"""Return the contents of the given template file, executed with the given
    local bindings.

    >>> from tempfile import NamedTemporaryFile
    >>> # On Windows, the name of a NamedTemporaryFile cannot be used to open
    >>> # the file for a second time if delete=True. Therefore, we have to
    >>> # manually handle closing and deleting this file to allow us to open
    >>> # the file by its name across all platforms.
    >>> f = NamedTemporaryFile(delete=False)
    >>> _ = f.write(
    ... br'''---
    ... % for i in range(int(x)):
    ... a pox on ${i} for epoxy
    ... % end
    ... ${120 +
    ...
    ...    3}
    ... abc
    ... ${"w\nx\nX\ny"}
    ... z
    ... ''')
    >>> f.flush()
    >>> result = expand(
    ...     f.name,
    ...     line_directive='//#sourceLocation(file: "%(file)s", ' + \
    ...                    'line: %(line)d)',
    ...     x=2
    ... ).replace(
    ...   '"%s"' % f.name.replace('\\', '/'), '"dummy.file"')
    >>> print(result, end='')
    //#sourceLocation(file: "dummy.file", line: 1)
    ---
    //#sourceLocation(file: "dummy.file", line: 3)
    a pox on 0 for epoxy
    //#sourceLocation(file: "dummy.file", line: 3)
    a pox on 1 for epoxy
    //#sourceLocation(file: "dummy.file", line: 5)
    123
    //#sourceLocation(file: "dummy.file", line: 8)
    abc
    w
    x
    X
    y
    //#sourceLocation(file: "dummy.file", line: 10)
    z
    >>> f.close()
    >>> os.remove(f.name)
    """
    with io.open(filename, encoding='utf-8') as f:
        t = parse_template(filename, f.read())
        d = os.getcwd()
        os.chdir(os.path.dirname(os.path.abspath(filename)))
        try:
            return execute_template(
                t, line_directive=line_directive, **local_bindings)
        finally:
            os.chdir(d)


def parse_template(filename, text=None):
    r"""Return an AST corresponding to the given template file.

    If text is supplied, it is assumed to be the contents of the file,
    as a string.

    >>> print(parse_template('dummy.file', text=
    ... '''% for x in [1, 2, 3]:
    ... %   if x == 1:
    ... literal1
    ... %   elif x > 1:  # add output line after this line to fix bug
    ... %     if x == 2:
    ... literal2
    ... %     end
    ... %   end
    ... % end
    ... '''))
    Block:
    [
        Code:
        {
            for x in [1, 2, 3]:
                __children__[0].execute(__context__)
        }
        [
            Block:
            [
                Code:
                {
                    if x == 1:
                        __children__[0].execute(__context__)
                    elif x > 1:  # add output line after this line to fix bug
                        __children__[1].execute(__context__)
                }
                [
                    Block:
                    [
                        Literal:
                        literal1
                    ]
                    Block:
                    [
                        Code:
                        {
                            if x == 2:
                                __children__[0].execute(__context__)
                        }
                        [
                            Block:
                            [
                                Literal:
                                literal2
                            ]
                        ]
                    ]
                ]
            ]
        ]
    ]

    >>> print(parse_template(
    ...     'dummy.file',
    ...     text='%for x in range(10):\n%  print(x)\n%end\njuicebox'))
    Block:
    [
        Code:
        {
            for x in range(10):
                __children__[0].execute(__context__)
        }
        [
            Block:
            [
                Code: {print(x)} []
            ]
        ]
        Literal:
        juicebox
    ]

    >>> print(parse_template('/dummy.file', text=
    ... '''Nothing
    ... % if x:
    ... %    for i in range(3):
    ... ${i}
    ... %    end
    ... % else:
    ... THIS SHOULD NOT APPEAR IN THE OUTPUT
    ... '''))
    Block:
    [
        Literal:
        Nothing
        Code:
        {
            if x:
                __children__[0].execute(__context__)
            else:
                __children__[1].execute(__context__)
        }
        [
            Block:
            [
                Code:
                {
                    for i in range(3):
                        __children__[0].execute(__context__)
                }
                [
                    Block:
                    [
                        Code: {(i)} []
                        Literal:
    <BLANKLINE>
                    ]
                ]
            ]
            Block:
            [
                Literal:
                THIS SHOULD NOT APPEAR IN THE OUTPUT
            ]
        ]
    ]

    >>> print(parse_template('dummy.file', text='''%
    ... %for x in y:
    ... %    print(y)
    ... '''))
    Block:
    [
        Code:
        {
            for x in y:
                __children__[0].execute(__context__)
        }
        [
            Block:
            [
                Code: {print(y)} []
            ]
        ]
    ]

    >>> print(parse_template('dummy.file', text='''%
    ... %if x:
    ... %    print(y)
    ... AAAA
    ... %else:
    ... BBBB
    ... '''))
    Block:
    [
        Code:
        {
            if x:
                __children__[0].execute(__context__)
            else:
                __children__[1].execute(__context__)
        }
        [
            Block:
            [
                Code: {print(y)} []
                Literal:
                AAAA
            ]
            Block:
            [
                Literal:
                BBBB
            ]
        ]
    ]

    >>> print(parse_template('dummy.file', text='''%
    ... %if x:
    ... %    print(y)
    ... AAAA
    ... %# This is a comment
    ... %else:
    ... BBBB
    ... '''))
    Block:
    [
        Code:
        {
            if x:
                __children__[0].execute(__context__)
            # This is a comment
            else:
                __children__[1].execute(__context__)
        }
        [
            Block:
            [
                Code: {print(y)} []
                Literal:
                AAAA
            ]
            Block:
            [
                Literal:
                BBBB
            ]
        ]
    ]

    >>> print(parse_template('dummy.file', text='''\
    ... %for x in y:
    ... AAAA
    ... %if x:
    ... BBBB
    ... %end
    ... CCCC
    ... '''))
    Block:
    [
        Code:
        {
            for x in y:
                __children__[0].execute(__context__)
        }
        [
            Block:
            [
                Literal:
                AAAA
                Code:
                {
                    if x:
                        __children__[0].execute(__context__)
                }
                [
                    Block:
                    [
                        Literal:
                        BBBB
                    ]
                ]
                Literal:
                CCCC
            ]
        ]
    ]
    """
    return Block(ParseContext(filename, text))


def execute_template(
        ast, line_directive=_default_line_directive, **local_bindings):
    r"""Return the text generated by executing the given template AST.

    Keyword arguments become local variable bindings in the execution context

    >>> root_directory = os.path.abspath('/')
    >>> file_name = (root_directory + 'dummy.file').replace('\\', '/')
    >>> ast = parse_template(file_name, text=
    ... '''Nothing
    ... % if x:
    ... %    for i in range(3):
    ... ${i}
    ... %    end
    ... % else:
    ... THIS SHOULD NOT APPEAR IN THE OUTPUT
    ... ''')
    >>> out = execute_template(ast,
    ... line_directive='//#sourceLocation(file: "%(file)s", line: %(line)d)',
    ... x=1)
    >>> out = out.replace(file_name, "DUMMY-FILE")
    >>> print(out, end="")
    //#sourceLocation(file: "DUMMY-FILE", line: 1)
    Nothing
    //#sourceLocation(file: "DUMMY-FILE", line: 4)
    0
    //#sourceLocation(file: "DUMMY-FILE", line: 4)
    1
    //#sourceLocation(file: "DUMMY-FILE", line: 4)
    2

    >>> ast = parse_template(file_name, text=
    ... '''Nothing
    ... % a = []
    ... % for x in range(3):
    ... %    a.append(x)
    ... % end
    ... ${a}
    ... ''')
    >>> out = execute_template(ast,
    ... line_directive='//#sourceLocation(file: "%(file)s", line: %(line)d)',
    ... x=1)
    >>> out = out.replace(file_name, "DUMMY-FILE")
    >>> print(out, end="")
    //#sourceLocation(file: "DUMMY-FILE", line: 1)
    Nothing
    //#sourceLocation(file: "DUMMY-FILE", line: 6)
    [0, 1, 2]

    >>> ast = parse_template(file_name, text=
    ... '''Nothing
    ... % a = []
    ... % for x in range(3):
    ... %    a.append(x)
    ... % end
    ... ${a}
    ... ''')
    >>> out = execute_template(ast,
    ...         line_directive='#line %(line)d "%(file)s"', x=1)
    >>> out = out.replace(file_name, "DUMMY-FILE")
    >>> print(out, end="")
    #line 1 "DUMMY-FILE"
    Nothing
    #line 6 "DUMMY-FILE"
    [0, 1, 2]
    """
    execution_context = ExecutionContext(
        line_directive=line_directive, **local_bindings)
    ast.execute(execution_context)
    return ''.join(execution_context.result_text)


def main():
    import argparse
    import sys

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Generate Your Boilerplate!', epilog='''
    A GYB template consists of the following elements:

      - Literal text which is inserted directly into the output

      - %% or $$ in literal text, which insert literal '%' and '$'
        symbols respectively.

      - Substitutions of the form ${<python-expression>}.  The Python
        expression is converted to a string and the result is inserted
        into the output.

      - Python code delimited by %{...}%.  Typically used to inject
        definitions (functions, classes, variable bindings) into the
        evaluation context of the template.  Common indentation is
        stripped, so you can add as much indentation to the beginning
        of this code as you like

      - Lines beginning with optional whitespace followed by a single
        '%' and Python code.  %-lines allow you to nest other
        constructs inside them.  To close a level of nesting, use the
        "%end" construct.

      - Lines beginning with optional whitespace and followed by a
        single '%' and the token "end", which close open constructs in
        %-lines.

    Example template:

          - Hello -
        %{
             x = 42
             def succ(a):
                 return a+1
        }%

        I can assure you that ${x} < ${succ(x)}

        % if int(y) > 7:
        %    for i in range(3):
        y is greater than seven!
        %    end
        % else:
        y is less than or equal to seven
        % end

          - The End. -

    When run with "gyb -Dy=9", the output is

          - Hello -

        I can assure you that 42 < 43

        y is greater than seven!
        y is greater than seven!
        y is greater than seven!

          - The End. -
'''
    )
    parser.add_argument(
        '-D', action='append', dest='defines', metavar='NAME=VALUE',
        default=[],
        help='''Bindings to be set in the template's execution context''')

    parser.add_argument(
        'file', type=str,
        help='Path to GYB template file (defaults to stdin)', nargs='?',
        default='-')
    parser.add_argument(
        '-o', dest='target', type=str,
        help='Output file (defaults to stdout)', default='-')
    parser.add_argument(
        '--test', action='store_true',
        default=False, help='Run a self-test')
    parser.add_argument(
        '--verbose-test', action='store_true',
        default=False, help='Run a verbose self-test')
    parser.add_argument(
        '--dump', action='store_true',
        default=False, help='Dump the parsed template to stdout')
    parser.add_argument(
        '--line-directive',
        default=_default_line_directive,
        help='''
             Line directive format string, which will be
             provided 2 substitutions, `%%(line)d` and `%%(file)s`.

             Example: `#sourceLocation(file: "%%(file)s", line: %%(line)d)`

             The default works automatically with the `line-directive` tool,
             which see for more information.
             ''')

    args = parser.parse_args(sys.argv[1:])

    if args.test or args.verbose_test:
        import doctest
        selfmod = sys.modules[__name__]
        if doctest.testmod(selfmod, verbose=args.verbose_test or None).failed:
            sys.exit(1)

    bindings = dict(x.split('=', 1) for x in args.defines)
    if args.file == '-':
        ast = parse_template('stdin', sys.stdin.read())
    else:
        with io.open(os.path.normpath(args.file), 'r', encoding='utf-8') as f:
            ast = parse_template(args.file, f.read())
    if args.dump:
        print(ast)

    # Allow the template to open files and import .py files relative to its own
    # directory
    saved_cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.abspath(args.file)))
    sys.path = ['.'] + sys.path
    result_text = execute_template(ast, args.line_directive, **bindings)

    if args.target == '-':
        sys.stdout.write(result_text)
    else:
        os.chdir(saved_cwd)
        with io.open(args.target, 'w', encoding='utf-8', newline='\n') as f:
            f.write(result_text)


if __name__ == '__main__':
    main()