From 505c99265ca6e9c75a3c9793b999cbdc46de1956 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 18 Jun 2019 17:56:33 +0200 Subject: [PATCH] docsite: remove lexers which have been fixed in Pygments 2.4.0 (#57508) * Remove lexers which have been fixed in Pygments 2.4.0. * Add Pygments >= 2.4.0 to test runner. * Fix pages that triggered lexer errors. Co-Authored-By: Sviatoslav Sydorenko --- docs/docsite/_extensions/pygments_lexer.py | 523 ------------------ docs/docsite/requirements.txt | 1 + .../rst/user_guide/playbooks_loops.rst | 2 +- docs/docsite/rst/user_guide/windows_dsc.rst | 4 +- docs/docsite/rst/user_guide/windows_usage.rst | 11 +- lib/ansible/plugins/lookup/passwordstore.py | 2 +- lib/ansible/plugins/lookup/skydive.py | 8 +- test/runner/requirements/constraints.txt | 1 + 8 files changed, 19 insertions(+), 533 deletions(-) diff --git a/docs/docsite/_extensions/pygments_lexer.py b/docs/docsite/_extensions/pygments_lexer.py index 837d2ea84d5..62c7fdfd76a 100644 --- a/docs/docsite/_extensions/pygments_lexer.py +++ b/docs/docsite/_extensions/pygments_lexer.py @@ -3,7 +3,6 @@ # # Copyright 2006-2017 by the Pygments team, see AUTHORS at # https://bitbucket.org/birkenfeld/pygments-main/raw/7941677dc77d4f2bf0bbd6140ade85a9454b8b80/AUTHORS -# Copyright by Kirill Simonov (original author of YAML lexer). # Copyright by Norman Richards (original author of JSON lexer). # # Licensed under BSD license: @@ -43,525 +42,6 @@ from pygments import token import re -class AnsibleYamlLexerContext(LexerContext): - """Indentation context for the YAML lexer.""" - - def __init__(self, *args, **kwds): - super(AnsibleYamlLexerContext, self).__init__(*args, **kwds) - self.indent_stack = [] - self.indent = -1 - self.next_indent = 0 - self.block_scalar_indent = None - - -class AnsibleYamlLexer(ExtendedRegexLexer): - """ - Lexer for `YAML `_, a human-friendly data serialization - language. - - .. versionadded:: 0.11 - """ - - name = 'YAML' - aliases = ['yaml'] - filenames = ['*.yaml', '*.yml'] - mimetypes = ['text/x-yaml'] - - def something(token_class): - """Do not produce empty tokens.""" - def callback(lexer, match, context): - text = match.group() - if not text: - return - yield match.start(), token_class, text - context.pos = match.end() - return callback - - def reset_indent(token_class): - """Reset the indentation levels.""" - def callback(lexer, match, context): - text = match.group() - context.indent_stack = [] - context.indent = -1 - context.next_indent = 0 - context.block_scalar_indent = None - yield match.start(), token_class, text - context.pos = match.end() - return callback - - def save_indent(token_class, start=False): - """Save a possible indentation level.""" - def callback(lexer, match, context): - text = match.group() - extra = '' - if start: - context.next_indent = len(text) - if context.next_indent < context.indent: - while context.next_indent < context.indent: - context.indent = context.indent_stack.pop() - if context.next_indent > context.indent: - extra = text[context.indent:] - text = text[:context.indent] - else: - context.next_indent += len(text) - if text: - yield match.start(), token_class, text - if extra: - yield match.start() + len(text), token_class.Error, extra - context.pos = match.end() - return callback - - def set_indent(token_class, implicit=False): - """Set the previously saved indentation level.""" - def callback(lexer, match, context): - text = match.group() - if context.indent < context.next_indent: - context.indent_stack.append(context.indent) - context.indent = context.next_indent - if not implicit: - context.next_indent += len(text) - yield match.start(), token_class, text - context.pos = match.end() - return callback - - def set_block_scalar_indent(token_class): - """Set an explicit indentation level for a block scalar.""" - def callback(lexer, match, context): - text = match.group() - context.block_scalar_indent = None - if not text: - return - increment = match.group(1) - if increment: - current_indent = max(context.indent, 0) - increment = int(increment) - context.block_scalar_indent = current_indent + increment - if text: - yield match.start(), token_class, text - context.pos = match.end() - return callback - - def parse_block_scalar_empty_line(indent_token_class, content_token_class): - """Process an empty line in a block scalar.""" - def callback(lexer, match, context): - text = match.group() - if (context.block_scalar_indent is None or - len(text) <= context.block_scalar_indent): - if text: - yield match.start(), indent_token_class, text - else: - indentation = text[:context.block_scalar_indent] - content = text[context.block_scalar_indent:] - yield match.start(), indent_token_class, indentation - yield (match.start() + context.block_scalar_indent, - content_token_class, content) - context.pos = match.end() - return callback - - def parse_block_scalar_indent(token_class): - """Process indentation spaces in a block scalar.""" - def callback(lexer, match, context): - text = match.group() - if context.block_scalar_indent is None: - if len(text) <= max(context.indent, 0): - context.stack.pop() - context.stack.pop() - return - context.block_scalar_indent = len(text) - else: - if len(text) < context.block_scalar_indent: - context.stack.pop() - context.stack.pop() - return - if text: - yield match.start(), token_class, text - context.pos = match.end() - return callback - - def parse_plain_scalar_indent(token_class): - """Process indentation spaces in a plain scalar.""" - def callback(lexer, match, context): - text = match.group() - if len(text) <= context.indent: - context.stack.pop() - context.stack.pop() - return - if text: - yield match.start(), token_class, text - context.pos = match.end() - return callback - - tokens = { - # the root rules - 'root': [ - # ignored whitespaces - (r'[ ]+(?=#|$)', token.Text), - # line breaks - (r'\n+', token.Text), - # a comment - (r'#[^\n]*', token.Comment.Single), - # the '%YAML' directive - (r'^%YAML(?=[ ]|$)', reset_indent(token.Name.Tag), 'yaml-directive'), - # the %TAG directive - (r'^%TAG(?=[ ]|$)', reset_indent(token.Name.Tag), 'tag-directive'), - # document start and document end indicators - (r'^(?:---|\.\.\.)(?=[ ]|$)', reset_indent(token.Name.Namespace), - 'block-line'), - # indentation spaces - (r'[ ]*(?!\s|$)', save_indent(token.Text, start=True), - ('block-line', 'indentation')), - ], - - # trailing whitespaces after directives or a block scalar indicator - 'ignored-line': [ - # ignored whitespaces - (r'[ ]+(?=#|$)', token.Text), - # a comment - (r'#[^\n]*', token.Comment.Single), - # line break - (r'\n', token.Text, '#pop:2'), - ], - - # the %YAML directive - 'yaml-directive': [ - # the version number - (r'([ ]+)([0-9]+\.[0-9]+)', - bygroups(token.Text, token.Number), 'ignored-line'), - ], - - # the %YAG directive - 'tag-directive': [ - # a tag handle and the corresponding prefix - (r'([ ]+)(!|![\w-]*!)' - r'([ ]+)(!|!?[\w;/?:@&=+$,.!~*\'()\[\]%-]+)', - bygroups(token.Text, token.Keyword.Type, token.Text, token.Keyword.Type), - 'ignored-line'), - ], - - # block scalar indicators and indentation spaces - 'indentation': [ - # trailing whitespaces are ignored - (r'[ ]*$', something(token.Text), '#pop:2'), - # whitespaces preceeding block collection indicators - (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(token.Text)), - # block collection indicators - (r'[?:-](?=[ ]|$)', set_indent(token.Punctuation.Indicator)), - # the beginning a block line - (r'[ ]*', save_indent(token.Text), '#pop'), - ], - - # an indented line in the block context - 'block-line': [ - # the line end - (r'[ ]*(?=#|$)', something(token.Text), '#pop'), - # whitespaces separating tokens - (r'[ ]+', token.Text), - # key with colon - (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''', - bygroups(token.Name.Tag, set_indent(token.Punctuation, implicit=True))), - # tags, anchors and aliases, - include('descriptors'), - # block collections and scalars - include('block-nodes'), - # flow collections and quoted scalars - include('flow-nodes'), - # a plain scalar - (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`-]|[?:-]\S)', - something(token.Name.Variable), - 'plain-scalar-in-block-context'), - ], - - # tags, anchors, aliases - 'descriptors': [ - # a full-form tag - (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', token.Keyword.Type), - # a tag in the form '!', '!suffix' or '!handle!suffix' - (r'!(?:[\w-]+!)?' - r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', token.Keyword.Type), - # an anchor - (r'&[\w-]+', token.Name.Label), - # an alias - (r'\*[\w-]+', token.Name.Variable), - ], - - # block collections and scalars - 'block-nodes': [ - # implicit key - (r':(?=[ ]|$)', set_indent(token.Punctuation.Indicator, implicit=True)), - # literal and folded scalars - (r'[|>]', token.Punctuation.Indicator, - ('block-scalar-content', 'block-scalar-header')), - ], - - # flow collections and quoted scalars - 'flow-nodes': [ - # a flow sequence - (r'\[', token.Punctuation.Indicator, 'flow-sequence'), - # a flow mapping - (r'\{', token.Punctuation.Indicator, 'flow-mapping'), - # a single-quoted scalar - (r'\'', token.String, 'single-quoted-scalar'), - # a double-quoted scalar - (r'\"', token.String, 'double-quoted-scalar'), - ], - - # the content of a flow collection - 'flow-collection': [ - # whitespaces - (r'[ ]+', token.Text), - # line breaks - (r'\n+', token.Text), - # a comment - (r'#[^\n]*', token.Comment.Single), - # simple indicators - (r'[?:,]', token.Punctuation.Indicator), - # tags, anchors and aliases - include('descriptors'), - # nested collections and quoted scalars - include('flow-nodes'), - # a plain scalar - (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`])', - something(token.Name.Variable), - 'plain-scalar-in-flow-context'), - ], - - # a flow sequence indicated by '[' and ']' - 'flow-sequence': [ - # include flow collection rules - include('flow-collection'), - # the closing indicator - (r'\]', token.Punctuation.Indicator, '#pop'), - ], - - # a flow mapping indicated by '{' and '}' - 'flow-mapping': [ - # key with colon - (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''', - bygroups(token.Name.Tag, token.Punctuation)), - # include flow collection rules - include('flow-collection'), - # the closing indicator - (r'\}', token.Punctuation.Indicator, '#pop'), - ], - - # block scalar lines - 'block-scalar-content': [ - # line break - (r'\n', token.Text), - # empty line - (r'^[ ]+$', - parse_block_scalar_empty_line(token.Text, token.Name.Constant)), - # indentation spaces (we may leave the state here) - (r'^[ ]*', parse_block_scalar_indent(token.Text)), - # line content - (r'[\S\t ]+', token.Name.Constant), - ], - - # the content of a literal or folded scalar - 'block-scalar-header': [ - # indentation indicator followed by chomping flag - (r'([1-9])?[+-]?(?=[ ]|$)', - set_block_scalar_indent(token.Punctuation.Indicator), - 'ignored-line'), - # chomping flag followed by indentation indicator - (r'[+-]?([1-9])?(?=[ ]|$)', - set_block_scalar_indent(token.Punctuation.Indicator), - 'ignored-line'), - ], - - # ignored and regular whitespaces in quoted scalars - 'quoted-scalar-whitespaces': [ - # leading and trailing whitespaces are ignored - (r'^[ ]+', token.Text), - (r'[ ]+$', token.Text), - # line breaks are ignored - (r'\n+', token.Text), - # other whitespaces are a part of the value - (r'[ ]+', token.Name.Variable), - ], - - # single-quoted scalars - 'single-quoted-scalar': [ - # include whitespace and line break rules - include('quoted-scalar-whitespaces'), - # escaping of the quote character - (r'\'\'', token.String.Escape), - # regular non-whitespace characters - (r'[^\s\']+', token.String), - # the closing quote - (r'\'', token.String, '#pop'), - ], - - # double-quoted scalars - 'double-quoted-scalar': [ - # include whitespace and line break rules - include('quoted-scalar-whitespaces'), - # escaping of special characters - (r'\\[0abt\tn\nvfre "\\N_LP]', token.String), - # escape codes - (r'\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})', - token.String.Escape), - # regular non-whitespace characters - (r'[^\s"\\]+', token.String), - # the closing quote - (r'"', token.String, '#pop'), - ], - - # the beginning of a new line while scanning a plain scalar - 'plain-scalar-in-block-context-new-line': [ - # empty lines - (r'^[ ]+$', token.Text), - # line breaks - (r'\n+', token.Text), - # document start and document end indicators - (r'^(?=---|\.\.\.)', something(token.Name.Namespace), '#pop:3'), - # indentation spaces (we may leave the block line state here) - (r'^[ ]*', parse_plain_scalar_indent(token.Text), '#pop'), - ], - - # a plain scalar in the block context - 'plain-scalar-in-block-context': [ - # the scalar ends with the ':' indicator - (r'[ ]*(?=:[ ]|:$)', something(token.Text), '#pop'), - # the scalar ends with whitespaces followed by a comment - (r'[ ]+(?=#)', token.Text, '#pop'), - # trailing whitespaces are ignored - (r'[ ]+$', token.Text), - # line breaks are ignored - (r'\n+', token.Text, 'plain-scalar-in-block-context-new-line'), - # other whitespaces are a part of the value - (r'[ ]+', token.Literal.Scalar.Plain), - # regular non-whitespace characters - (r'(?::(?!\s)|[^\s:])+', token.Literal.Scalar.Plain), - ], - - # a plain scalar is the flow context - 'plain-scalar-in-flow-context': [ - # the scalar ends with an indicator character - (r'[ ]*(?=[,:?\[\]{}])', something(token.Text), '#pop'), - # the scalar ends with a comment - (r'[ ]+(?=#)', token.Text, '#pop'), - # leading and trailing whitespaces are ignored - (r'^[ ]+', token.Text), - (r'[ ]+$', token.Text), - # line breaks are ignored - (r'\n+', token.Text), - # other whitespaces are a part of the value - (r'[ ]+', token.Name.Variable), - # regular non-whitespace characters - (r'[^\s,:?\[\]{}]+', token.Name.Variable), - ], - - } - - def get_tokens_unprocessed(self, text=None, context=None): - if context is None: - context = AnsibleYamlLexerContext(text, 0) - return super(AnsibleYamlLexer, self).get_tokens_unprocessed(text, context) - - -class AnsibleDjangoLexer(RegexLexer): - """ - Generic `django `_ - and `jinja `_ template lexer. - - It just highlights django/jinja code between the preprocessor directives, - other data is left untouched by the lexer. - """ - - name = 'Django/Jinja' - aliases = ['django', 'jinja'] - mimetypes = ['application/x-django-templating', 'application/x-jinja'] - - flags = re.M | re.S - - tokens = { - 'root': [ - (r'[^{]+', token.Other), - (r'\{\{', token.Comment.Preproc, 'var'), - # jinja/django comments - (r'\{[*#].*?[*#]\}', token.Comment), - # django comments - (r'(\{%)(-?\s*)(comment)(\s*-?)(%\})(.*?)' - r'(\{%)(-?\s*)(endcomment)(\s*-?)(%\})', - bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Comment.Preproc, - token.Comment, token.Comment.Preproc, token.Text, token.Keyword, token.Text, - token.Comment.Preproc)), - # raw jinja blocks - (r'(\{%)(-?\s*)(raw)(\s*-?)(%\})(.*?)' - r'(\{%)(-?\s*)(endraw)(\s*-?)(%\})', - bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Comment.Preproc, - token.Text, token.Comment.Preproc, token.Text, token.Keyword, token.Text, - token.Comment.Preproc)), - # filter blocks - (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_]\w*)', - bygroups(token.Comment.Preproc, token.Text, token.Keyword, token.Text, token.Name.Function), - 'block'), - (r'(\{%)(-?\s*)([a-zA-Z_]\w*)', - bygroups(token.Comment.Preproc, token.Text, token.Keyword), 'block'), - (r'\{', token.Other) - ], - 'varnames': [ - (r'(\|)(\s*)([a-zA-Z_]\w*)', - bygroups(token.Operator, token.Text, token.Name.Function)), - (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_]\w*)', - bygroups(token.Keyword, token.Text, token.Keyword, token.Text, token.Name.Function)), - (r'(_|true|false|none|True|False|None)\b', token.Keyword.Pseudo), - (r'(in|as|reversed|recursive|not|and|or|is|if|else|import|' - r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b', - token.Keyword), - (r'(loop|block|super|forloop)\b', token.Name.Builtin), - (r'[a-zA-Z_][\w-]*', token.Name.Variable), - (r'\.\w+', token.Name.Variable), - (r':?"(\\\\|\\"|[^"])*"', token.String.Double), - (r":?'(\\\\|\\'|[^'])*'", token.String.Single), - (r'([{}()\[\]+\-*/%,:~]|[><=]=?|!=)', token.Operator), - (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" - r"0[xX][0-9a-fA-F]+[Ll]?", token.Number), - ], - 'var': [ - (r'\s+', token.Text), - (r'(-?)(\}\})', bygroups(token.Text, token.Comment.Preproc), '#pop'), - include('varnames') - ], - 'block': [ - (r'\s+', token.Text), - (r'(-?)(%\})', bygroups(token.Text, token.Comment.Preproc), '#pop'), - include('varnames'), - (r'.', token.Punctuation) - ] - } - - def analyse_text(text): - rv = 0.0 - if re.search(r'\{%\s*(block|extends)', text) is not None: - rv += 0.4 - if re.search(r'\{%\s*if\s*.*?%\}', text) is not None: - rv += 0.1 - if re.search(r'\{\{.*?\}\}', text) is not None: - rv += 0.1 - return rv - - -class AnsibleYamlJinjaLexer(DelegatingLexer): - """ - Subclass of the `DjangoLexer` that highlights unlexed data with the - `AnsibleYamlLexer`. - - Commonly used in Saltstack salt states. - - .. versionadded:: 2.0 - """ - - name = 'YAML+Jinja' - aliases = ['yaml+jinja'] - filenames = ['*.sls'] - mimetypes = ['text/x-yaml+jinja'] - - def __init__(self, **options): - super(AnsibleYamlJinjaLexer, self).__init__(AnsibleYamlLexer, AnsibleDjangoLexer, **options) - - class AnsibleOutputPrimaryLexer(RegexLexer): name = 'Ansible-output-primary' @@ -698,9 +178,6 @@ def setup(app): See http://www.sphinx-doc.org/en/stable/extdev/index.html#dev-extensions. """ for lexer in [ - AnsibleDjangoLexer(startinline=True), - AnsibleYamlLexer(startinline=True), - AnsibleYamlJinjaLexer(startinline=True), AnsibleOutputLexer(startinline=True) ]: app.add_lexer(lexer.name, lexer) diff --git a/docs/docsite/requirements.txt b/docs/docsite/requirements.txt index 632f07bd64a..f5492102b71 100644 --- a/docs/docsite/requirements.txt +++ b/docs/docsite/requirements.txt @@ -4,3 +4,4 @@ PyYAML rstcheck sphinx sphinx-notfound-page +Pygments >= 2.4.0 diff --git a/docs/docsite/rst/user_guide/playbooks_loops.rst b/docs/docsite/rst/user_guide/playbooks_loops.rst index 551384205bb..4dbfaa66aee 100644 --- a/docs/docsite/rst/user_guide/playbooks_loops.rst +++ b/docs/docsite/rst/user_guide/playbooks_loops.rst @@ -33,7 +33,7 @@ Comparing ``loop`` and ``with_*`` you would need:: - loop: [1, [2,3] ,4] | flatten(1) + loop: "{{ [1, [2,3] ,4] | flatten(1) }}" * Any ``with_*`` statement that requires using ``lookup`` within a loop should not be converted to use the ``loop`` keyword. For example, instead of doing: diff --git a/docs/docsite/rst/user_guide/windows_dsc.rst b/docs/docsite/rst/user_guide/windows_dsc.rst index 167256ae66c..af4a02de768 100644 --- a/docs/docsite/rst/user_guide/windows_dsc.rst +++ b/docs/docsite/rst/user_guide/windows_dsc.rst @@ -105,7 +105,9 @@ fail if the option name is incorrect, a mandatory option is not set, or the value is not a valid choice. When running Ansible with a verbosity level of 3 or more (``-vvv``), the return value will contain the possible invocation options based on the ``resource_name`` specified. Here is an example of the -invocation output for the above ``Registry`` task:: +invocation output for the above ``Registry`` task: + +.. code-block:: ansible-output changed: [2016] => { "changed": true, diff --git a/docs/docsite/rst/user_guide/windows_usage.rst b/docs/docsite/rst/user_guide/windows_usage.rst index d95afbd2b9f..d49d67e6096 100644 --- a/docs/docsite/rst/user_guide/windows_usage.rst +++ b/docs/docsite/rst/user_guide/windows_usage.rst @@ -402,9 +402,7 @@ The YAML specification considers the following `escape sequences - + {{ lookup('skydive', filter={'query': "G.V().Has('Name', 'sumit-VirtualBox')"}) }} - name: return all the skydive metdata having parameter Name set_fact: - skydive: "{{ lookup('skydive', filter={'query': \"G.V().Has('Name')\"}, - provider={'endpoint': 'localhost:8082', 'username': 'admin', 'password': 'password'}) }}" + skydive: >- + {{ lookup('skydive', filter={'query': "G.V().Has('Name')"}, + provider={'endpoint': 'localhost:8082', 'username': 'admin', 'password': 'password'}) }} """ RETURN = """ diff --git a/test/runner/requirements/constraints.txt b/test/runner/requirements/constraints.txt index 37616763dbc..03800963ff5 100644 --- a/test/runner/requirements/constraints.txt +++ b/test/runner/requirements/constraints.txt @@ -5,6 +5,7 @@ urllib3 < 1.24 ; python_version < '2.7' # urllib3 1.24 and later require python pywinrm >= 0.3.0 # message encryption support sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7 or later sphinx < 1.8 ; python_version >= '2.7' # sphinx 1.8 and later are currently incompatible with rstcheck 3.3 +pygments >= 2.4.0 # Pygments 2.4.0 includes bugfixes for YAML and YAML+Jinja lexers wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later yamllint != 1.8.0, < 1.14.0 ; python_version < '2.7' # yamllint 1.8.0 and 1.14.0+ require python 2.7+ pycrypto >= 2.6 # Need features found in 2.6 and greater