Refactor parsing of CLI args so that we can modify them in the base class

Implement tag and skip_tag handling in the CLI() class.  Change tag and
skip_tag command line options to be accepted multiple times on the CLI
and add them together rather than overwrite.

* Make it configurable whether to merge or overwrite multiple --tags arguments
* Make the base CLI class an abstractbaseclass so we can implement
  functionality in parse() but still make subclasses implement it.
* Deprecate the overwrite feature of --tags with a message that the
  default will change in 2.4 and go away in 2.5.

* Add documentation for merge_multiple_cli_flags
* Fix galaxy search so its tags argument does not conflict with generic tags
* Unit tests and more integration tests for tags
This commit is contained in:
Toshio Kuratomi 2016-09-29 14:14:02 -07:00 committed by Brian Coca
parent 9962245b92
commit 1efe782b46
15 changed files with 162 additions and 77 deletions

View file

@ -179,17 +179,6 @@ different locations::
Most users will not need to use this feature. See :doc:`developing_plugins` for more details
.. _stdout_callback:
stdout_callback
===============
.. versionadded:: 2.0
This setting allows you to override the default stdout callback for ansible-playbook::
stdout_callback = skippy
.. _callback_whitelist:
callback_whitelist
@ -523,6 +512,23 @@ different locations::
Most users will not need to use this feature. See :doc:`developing_plugins` for more details
.. _merge_multiple_cli_tags:
merge_multiple_cli_tags
=======================
.. versionadded:: 2.3
This allows changing how multiple --tags and --skip-tags arguments are handled
on the command line. In Ansible up to and including 2.3, specifying --tags
more than once will only take the last value of --tags. Setting this config
value to True will mean that all of the --tags options will be merged
together. The same holds true for --skip-tags.
.. note:: The default value for this in 2.3 is False. In 2.4, the
default value will be True. After 2.4, the option is going away.
Multiple --tags and multiple --skip-tags will always be merged together.
.. _module_set_locale:
module_set_locale
@ -705,6 +711,17 @@ The default value for this setting is only for certain package managers, but it
Currently, this is only supported for modules that have a name parameter, and only when the item is the
only thing being passed to the parameter.
.. _stdout_callback:
stdout_callback
===============
.. versionadded:: 2.0
This setting allows you to override the default stdout callback for ansible-playbook::
stdout_callback = skippy
.. _cfg_strategy_plugins:
strategy_plugins

View file

@ -261,6 +261,11 @@
# set to 0 for unlimited (RAM may suffer!).
#max_diff_size = 1048576
# This controls how ansible handles multiple --tags and --skip-tags arguments
# on the CLI. If this is True then multiple arguments are merged together. If
# it is False, then the last specified argument is used and the others are ignored.
#merge_multiple_cli_flags = False
[privilege_escalation]
#become=True
#become_method=sudo

View file

@ -30,9 +30,11 @@ import re
import getpass
import signal
import subprocess
from abc import ABCMeta, abstractmethod
from ansible.release import __version__
from ansible import constants as C
from ansible.compat.six import with_metaclass
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.module_utils._text import to_bytes, to_text
@ -89,7 +91,7 @@ class InvalidOptsParser(SortedOptParser):
pass
class CLI(object):
class CLI(with_metaclass(ABCMeta, object)):
''' code behind bin/ansible* programs '''
VALID_ACTIONS = ['No Actions']
@ -144,10 +146,13 @@ class CLI(object):
fn = getattr(self, "execute_%s" % self.action)
fn()
def parse(self):
raise Exception("Need to implement!")
@abstractmethod
def run(self):
"""Run the ansible command
Subclasses must implement this method. It does the actual work of
running an Ansible command.
"""
if self.options.verbosity > 0:
if C.CONFIG_FILE:
@ -155,7 +160,6 @@ class CLI(object):
else:
display.display(u"No config file found; using defaults")
@staticmethod
def ask_vault_passwords(ask_new_vault_pass=False, rekey=False):
''' prompt for vault password and/or password change '''
@ -314,9 +318,9 @@ class CLI(object):
action="callback", callback=CLI.expand_tilde, type=str)
if subset_opts:
parser.add_option('-t', '--tags', dest='tags', default='all',
parser.add_option('-t', '--tags', dest='tags', default=[], action='append',
help="only run plays and tasks tagged with these values")
parser.add_option('--skip-tags', dest='skip_tags',
parser.add_option('--skip-tags', dest='skip_tags', default=[], action='append',
help="only run plays and tasks whose tags do not match these values")
if output_opts:
@ -405,6 +409,55 @@ class CLI(object):
return parser
@abstractmethod
def parse(self):
"""Parse the command line args
This method parses the command line arguments. It uses the parser
stored in the self.parser attribute and saves the args and options in
self.args and self.options respectively.
Subclasses need to implement this method. They will usually create
a base_parser, add their own options to the base_parser, and then call
this method to do the actual parsing. An implementation will look
something like this::
def parse(self):
parser = super(MyCLI, self).base_parser(usage="My Ansible CLI", inventory_opts=True)
parser.add_option('--my-option', dest='my_option', action='store')
self.parser = parser
super(MyCLI, self).parse()
# If some additional transformations are needed for the
# arguments and options, do it here.
"""
self.options, self.args = self.parser.parse_args(self.args[1:])
if hasattr(self.options, 'tags') and not self.options.tags:
# optparse defaults does not do what's expected
self.options.tags = ['all']
if hasattr(self.options, 'tags') and self.options.tags:
if not C.MERGE_MULTIPLE_CLI_TAGS:
if len(self.options.tags) > 1:
display.deprecated('Specifying --tags multiple times on the command line currently uses the last specified value. In 2.4, values will be merged instead. Set merge_multiple_cli_tags=True in ansible.cfg to get this behavior now.', version=2.5, removed=False)
self.options.tags = [self.options.tags[-1]]
tags = set()
for tag_set in self.options.tags:
for tag in tag_set.split(u','):
tags.add(tag.strip())
self.options.tags = list(tags)
if hasattr(self.options, 'skip_tags') and self.options.skip_tags:
if not C.MERGE_MULTIPLE_CLI_TAGS:
if len(self.options.skip_tags) > 1:
display.deprecated('Specifying --skip-tags multiple times on the command line currently uses the last specified value. In 2.4, values will be merged instead. Set merge_multiple_cli_tags=True in ansible.cfg to get this behavior now.', version=2.5, removed=False)
self.options.skip_tags = [self.options.skip_tags[-1]]
skip_tags = set()
for tag_set in self.options.skip_tags:
for tag in tag_set.split(u','):
skip_tags.add(tag.strip())
self.options.skip_tags = list(skip_tags)
@staticmethod
def version(prog):
''' return ansible version '''

View file

@ -72,7 +72,7 @@ class AdHocCLI(CLI):
help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME,
default=C.DEFAULT_MODULE_NAME)
self.options, self.args = self.parser.parse_args(self.args[1:])
super(AdHocCLI, self).parse()
if len(self.args) < 1:
raise AnsibleOptionsError("Missing target hosts")
@ -82,8 +82,6 @@ class AdHocCLI(CLI):
display.verbosity = self.options.verbosity
self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True)
return True
def _play_ds(self, pattern, async, poll):
check_raw = self.options.module_name in ('command', 'win_command', 'shell', 'win_shell', 'script', 'raw')
return dict(

View file

@ -94,13 +94,12 @@ class ConsoleCLI(CLI, cmd.Cmd):
help="one-step-at-a-time: confirm each task before running")
self.parser.set_defaults(cwd='*')
self.options, self.args = self.parser.parse_args(self.args[1:])
super(AdHocCLI, self).parse()
display.verbosity = self.options.verbosity
self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True)
return True
def get_names(self):
return dir(self)

View file

@ -60,7 +60,8 @@ class DocCLI(CLI):
self.parser.add_option("-s", "--snippet", action="store_true", default=False, dest='show_snippet',
help='Show playbook snippet for specified module(s)')
self.options, self.args = self.parser.parse_args(self.args[1:])
super(DocCLI, self).parse()
display.verbosity = self.options.verbosity
def run(self):

View file

@ -100,7 +100,7 @@ class GalaxyCLI(CLI):
elif self.action == "search":
self.parser.set_usage("usage: %prog search [searchterm1 searchterm2] [--galaxy-tags galaxy_tag1,galaxy_tag2] [--platforms platform1,platform2] [--author username]")
self.parser.add_option('--platforms', dest='platforms', help='list of OS platforms to filter by')
self.parser.add_option('--galaxy-tags', dest='tags', help='list of galaxy tags to filter by')
self.parser.add_option('--galaxy-tags', dest='galaxy_tags', help='list of galaxy tags to filter by')
self.parser.add_option('--author', dest='author', help='GitHub username')
elif self.action == "setup":
self.parser.set_usage("usage: %prog setup [options] source github_user github_repo secret")
@ -120,10 +120,10 @@ class GalaxyCLI(CLI):
if self.action in ("init","install"):
self.parser.add_option('-f', '--force', dest='force', action='store_true', default=False, help='Force overwriting an existing role')
self.options, self.args = self.parser.parse_args(self.args[1:])
super(GalaxyCLI, self).parse()
display.verbosity = self.options.verbosity
self.galaxy = Galaxy(self.options)
return True
def run(self):
@ -505,11 +505,11 @@ class GalaxyCLI(CLI):
terms.append(self.args.pop())
search = '+'.join(terms[::-1])
if not search and not self.options.platforms and not self.options.tags and not self.options.author:
if not search and not self.options.platforms and not self.options.galaxy_tags and not self.options.author:
raise AnsibleError("Invalid query. At least one search term, platform, galaxy tag or author must be provided.")
response = self.api.search_roles(search, platforms=self.options.platforms,
tags=self.options.tags, author=self.options.author, page_size=page_size)
tags=self.options.galaxy_tags, author=self.options.author, page_size=page_size)
if response['count'] == 0:
display.display("No roles match your search.", color=C.COLOR_ERROR)

View file

@ -75,10 +75,8 @@ class PlaybookCLI(CLI):
parser.add_option('--start-at-task', dest='start_at_task',
help="start the playbook at the task matching this name")
self.options, self.args = parser.parse_args(self.args[1:])
self.parser = parser
super(PlaybookCLI, self).parse()
if len(self.args) == 0:
raise AnsibleOptionsError("You must specify a playbook file to run")

View file

@ -30,6 +30,7 @@ import time
from ansible.errors import AnsibleOptionsError
from ansible.cli import CLI
from ansible.module_utils._text import to_native
from ansible.plugins import module_loader
from ansible.utils.cmd_functions import run_cmd
@ -100,7 +101,7 @@ class PullCLI(CLI):
# for pull we don't wan't a default
self.parser.set_defaults(inventory=None)
self.options, self.args = self.parser.parse_args(self.args[1:])
super(PullCLI, self).parse()
if not self.options.dest:
hostname = socket.getfqdn()
@ -219,9 +220,9 @@ class PullCLI(CLI):
if self.options.ask_sudo_pass or self.options.ask_su_pass or self.options.become_ask_pass:
cmd += ' --ask-become-pass'
if self.options.skip_tags:
cmd += ' --skip-tags "%s"' % self.options.skip_tags
cmd += ' --skip-tags "%s"' % to_native(u','.join(self.options.skip_tags))
if self.options.tags:
cmd += ' -t "%s"' % self.options.tags
cmd += ' -t "%s"' % to_native(u','.join(self.options.tags))
if self.options.subset:
cmd += ' -l "%s"' % self.options.subset
else:

View file

@ -70,7 +70,8 @@ class VaultCLI(CLI):
elif self.action == "rekey":
self.parser.set_usage("usage: %prog rekey [options] file_name")
self.options, self.args = self.parser.parse_args(self.args[1:])
super(VaultCLI, self).parse()
display.verbosity = self.options.verbosity
can_output = ['encrypt', 'decrypt']

View file

@ -151,6 +151,11 @@ DEFAULTS='defaults'
DEPRECATED_HOST_LIST = get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts', ispath=True)
# this is not used since 0.5 but people might still have in config
DEFAULT_PATTERN = get_config(p, DEFAULTS, 'pattern', None, None)
# If --tags or --skip-tags is given multiple times on the CLI and this is
# True, merge the lists of tags together. If False, let the last argument
# overwrite any previous ones. Behaviour is overwrite through 2.2. 2.3
# overwrites but prints deprecation. 2.4 the default is to merge.
MERGE_MULTIPLE_CLI_TAGS = get_config(p, DEFAULTS, 'merge_multiple_cli_tags', 'ANSIBLE_MERGE_MULTIPLE_CLI_TAGS', False, boolean=True)
#### GENERALLY CONFIGURABLE THINGS ####
DEFAULT_DEBUG = get_config(p, DEFAULTS, 'debug', 'ANSIBLE_DEBUG', False, boolean=True)

View file

@ -284,23 +284,16 @@ class PlayContext(Base):
if hasattr(options, 'timeout') and options.timeout:
self.timeout = int(options.timeout)
# get the tag info from options, converting a comma-separated list
# of values into a proper list if need be. We check to see if the
# options have the attribute, as it is not always added via the CLI
# get the tag info from options. We check to see if the options have
# the attribute, as it is not always added via the CLI
if hasattr(options, 'tags'):
if isinstance(options.tags, list):
self.only_tags.update(options.tags)
elif isinstance(options.tags, string_types):
self.only_tags.update(options.tags.split(','))
self.only_tags.update(options.tags)
if len(self.only_tags) == 0:
self.only_tags = set(['all'])
if hasattr(options, 'skip_tags'):
if isinstance(options.skip_tags, list):
self.skip_tags.update(options.skip_tags)
elif isinstance(options.skip_tags, string_types):
self.skip_tags.update(options.skip_tags.split(','))
self.skip_tags.update(options.skip_tags)
def set_task_and_variable_override(self, task, variables, templar):
'''

View file

@ -193,12 +193,19 @@ test_win_group3: setup
ansible-playbook test_win_group3.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
test_tags: setup
# Run these using en_US.UTF-8 because list-tasks is a user output function and so it tailors its output to the user's locale. For unicode tags, this means replacing non-ascii chars with "?"
# Run everything by default
[ "$$(ansible-playbook --list-tasks test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_without_tag TAGS: []" ]
[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
# Run the exact tags, and always
[ "$$(ansible-playbook --list-tasks --tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always]" ]
[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always]" ]
# Skip one tag
[ "$$(ansible-playbook --list-tasks --skip-tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_without_tag TAGS: []" ]
[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --skip-tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
# Skip a unicode tag
[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --skip-tags くらとみ test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
# Run just a unicode tag and always
[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags くらとみ test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ]" ]
# Run a tag from a list of tags and always
[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags café test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press]" ]
blocks: setup
# remove old output log

View file

@ -10,6 +10,14 @@
- name: Task_with_always_tag
debug: msg=
tags: always
- name: Task_with_unicode_tag
debug: msg=
tags: くらとみ
- name: Task_with_list_of_tags
debug: msg=
tags:
- café
- press
- name: Task_without_tag
debug: msg=

View file

@ -154,12 +154,11 @@ class TestGalaxy(unittest.TestCase):
def run_parse_common(self, galaxycli_obj, action):
with patch.object(ansible.cli.SortedOptParser, "set_usage") as mocked_usage:
galaxy_parser = galaxycli_obj.parse()
galaxycli_obj.parse()
# checking that the common results of parse() for all possible actions have been created/called
self.assertTrue(galaxy_parser)
self.assertTrue(isinstance(galaxycli_obj.parser, ansible.cli.SortedOptParser))
self.assertTrue(isinstance(galaxycli_obj.galaxy, ansible.galaxy.Galaxy))
self.assertIsInstance(galaxycli_obj.parser, ansible.cli.SortedOptParser)
self.assertIsInstance(galaxycli_obj.galaxy, ansible.galaxy.Galaxy)
if action in ['import', 'delete']:
formatted_call = 'usage: %prog ' + action + ' [options] github_user github_repo'
elif action == 'info':
@ -194,61 +193,61 @@ class TestGalaxy(unittest.TestCase):
# testing action 'delete'
gc = GalaxyCLI(args=["delete", "-c"])
self.run_parse_common(gc, "delete")
self.assertTrue(gc.options.verbosity==0)
self.assertEqual(gc.options.verbosity, 0)
# testing action 'import'
gc = GalaxyCLI(args=["import", "-c"])
self.run_parse_common(gc, "import")
self.assertTrue(gc.options.wait==True)
self.assertTrue(gc.options.reference==None)
self.assertTrue(gc.options.check_status==False)
self.assertTrue(gc.options.verbosity==0)
self.assertEqual(gc.options.wait, True)
self.assertEqual(gc.options.reference, None)
self.assertEqual(gc.options.check_status, False)
self.assertEqual(gc.options.verbosity, 0)
# testing action 'info'
gc = GalaxyCLI(args=["info", "-c"])
self.run_parse_common(gc, "info")
self.assertTrue(gc.options.offline==False)
self.assertEqual(gc.options.offline, False)
# testing action 'init'
gc = GalaxyCLI(args=["init", "-c"])
self.run_parse_common(gc, "init")
self.assertTrue(gc.options.offline==False)
self.assertTrue(gc.options.force==False)
self.assertEqual(gc.options.offline, False)
self.assertEqual(gc.options.force, False)
# testing action 'install'
gc = GalaxyCLI(args=["install", "-c"])
self.run_parse_common(gc, "install")
self.assertTrue(gc.options.ignore_errors==False)
self.assertTrue(gc.options.no_deps==False)
self.assertTrue(gc.options.role_file==None)
self.assertTrue(gc.options.force==False)
self.assertEqual(gc.options.ignore_errors, False)
self.assertEqual(gc.options.no_deps, False)
self.assertEqual(gc.options.role_file, None)
self.assertEqual(gc.options.force, False)
# testing action 'list'
gc = GalaxyCLI(args=["list", "-c"])
self.run_parse_common(gc, "list")
self.assertTrue(gc.options.verbosity==0)
self.assertEqual(gc.options.verbosity, 0)
# testing action 'login'
gc = GalaxyCLI(args=["login", "-c"])
self.run_parse_common(gc, "login")
self.assertTrue(gc.options.verbosity==0)
self.assertTrue(gc.options.token==None)
self.assertEqual(gc.options.verbosity, 0)
self.assertEqual(gc.options.token, None)
# testing action 'remove'
gc = GalaxyCLI(args=["remove", "-c"])
self.run_parse_common(gc, "remove")
self.assertTrue(gc.options.verbosity==0)
self.assertEqual(gc.options.verbosity, 0)
# testing action 'search'
gc = GalaxyCLI(args=["search", "-c"])
self.run_parse_common(gc, "search")
self.assertTrue(gc.options.platforms==None)
self.assertTrue(gc.options.tags==None)
self.assertTrue(gc.options.author==None)
self.assertEqual(gc.options.platforms, None)
self.assertEqual(gc.options.galaxy_tags, None)
self.assertEqual(gc.options.author, None)
# testing action 'setup'
gc = GalaxyCLI(args=["setup", "-c"])
self.run_parse_common(gc, "setup")
self.assertTrue(gc.options.verbosity==0)
self.assertTrue(gc.options.remove_id==None)
self.assertTrue(gc.options.setup_list==False)
self.assertEqual(gc.options.verbosity, 0)
self.assertEqual(gc.options.remove_id, None)
self.assertEqual(gc.options.setup_list, False)