cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

FEATURE: adding variable serial batches

Browse source 
This feature changes the scalar value of `serial:` to a list, which
allows users to specify a list of values, so batches can be ramped
up (commonly called "canary" setups):

- hosts: all
  serial: [1, 5, 10, "100%"]
  tasks:
  ...
This commit is contained in:
James Cammarata 2016-08-04 00:05:30 -05:00
parent 3a83333ef2
commit 159aa26b36
8 changed files with 176 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -13,6 +13,7 @@ Ansible Changes By Release
- ansible_date_time.iso8601 (changed to UTC instead of local time)
- ansible_distribution (now uses OS caption string, e.g.: "Microsoft Windows Server 2012 R2 Standard", version is still available on ansible_distribution_version)
- ansible_totalmem (renamed to ansible_memtotal_mb, units changed to MB instead of bytes)
- Added the ability to specify serial batches as a list (`serial: [1, 5, 10]`), which allows for so-called "canary" actions in one play.
####New Modules:
- asa

30
docsite/rst/playbooks_delegation.rst
View file

@ -42,6 +42,36 @@ play, in order to determine the number of hosts per pass::
If the number of hosts does not divide equally into the number of passes, the final pass will contain the remainder.
As of Ansible 2.2, the batch sizes can be specified as a list, as follows::
- name: test play
hosts: webservers
serial:
- 1
- 5
- 10
In the above example, the first batch would contain a single host, the next would contain 5 hosts, and (if there are any hosts left),
every following batch would contain 10 hosts until all available hosts are used.
It is also possible to list multiple batche sizes as percentages::
- name: test play
hosts: webservers
serial:
- "10%"
- "20%"
- "100%"
You can also mix and match the values::
- name: test play
hosts: webservers
serial:
- 1
- 5
- "20%"
.. note::
No matter how small the percentage, the number of hosts per pass will always be 1 or greater.

49
lib/ansible/executor/playbook_executor.py
View file

@ -27,6 +27,7 @@ from ansible import constants as C
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.playbook import Playbook
from ansible.template import Templar
from ansible.utils.helpers import pct_to_int
from ansible.utils.path import makedirs_safe
from ansible.utils.unicode import to_unicode, to_str
@ -228,27 +229,28 @@ class PlaybookExecutor:
# make sure we have a unique list of hosts
all_hosts = self._inventory.get_hosts(play.hosts)
all_hosts_len = len(all_hosts)
# check to see if the serial number was specified as a percentage,
# and convert it to an integer value based on the number of hosts
if isinstance(play.serial, string_types) and play.serial.endswith('%'):
serial_pct = int(play.serial.replace("%",""))
serial = int((serial_pct/100.0) * len(all_hosts)) or 1
else:
if play.serial is None:
serial = -1
# the serial value can be listed as a scalar or a list of
# scalars, so we make sure it's a list here
serial_batch_list = play.serial
if len(serial_batch_list) == 0:
serial_batch_list = [-1]
cur_item = 0
serialized_batches = []
while len(all_hosts) > 0:
# get the serial value from current item in the list
serial = pct_to_int(serial_batch_list[cur_item], all_hosts_len)
# if the serial count was not specified or is invalid, default to
# a list of all hosts, otherwise grab a chunk of the hosts equal
# to the current serial item size
if serial <= 0:
serialized_batches.append(all_hosts)
break
else:
serial = int(play.serial)
# if the serial count was not specified or is invalid, default to
# a list of all hosts, otherwise split the list of hosts into chunks
# which are based on the serial size
if serial <= 0:
return [all_hosts]
else:
serialized_batches = []
while len(all_hosts) > 0:
play_hosts = []
for x in range(serial):
if len(all_hosts) > 0:
@ -256,7 +258,14 @@ class PlaybookExecutor:
serialized_batches.append(play_hosts)
return serialized_batches
# increment the current batch list item number, and if we've hit
# the end keep using the last element until we've consumed all of
# the hosts in the inventory
cur_item += 1
if cur_item > len(serial_batch_list) - 1:
cur_item = len(serial_batch_list) - 1
return serialized_batches
def _generate_retry_inventory(self, retry_path, replay_hosts):
'''

16
lib/ansible/executor/task_queue_manager.py
View file

@ -34,6 +34,7 @@ from ansible.plugins import callback_loader, strategy_loader, module_loader
from ansible.template import Templar
from ansible.vars.hostvars import HostVars
from ansible.plugins.callback import CallbackBase
from ansible.utils.helpers import pct_to_int
from ansible.utils.unicode import to_unicode
from ansible.compat.six import string_types
@ -225,8 +226,19 @@ class TaskQueueManager:
)
# Fork # of forks, # of hosts or serial, whichever is lowest
contenders = [self._options.forks, play.serial, len(self._inventory.get_hosts(new_play.hosts))]
contenders = [ v for v in contenders if v is not None and v > 0 ]
num_hosts = len(self._inventory.get_hosts(new_play.hosts))
max_serial = 0
if play.serial:
# the play has not been post_validated here, so we may need
# to convert the scalar value to a list at this point
serial_items = play.serial
if not isinstance(serial_items, list):
serial_items = [serial_items]
max_serial = max([pct_to_int(x, num_hosts) for x in serial_items])
contenders = [self._options.forks, max_serial, num_hosts]
contenders = [v for v in contenders if v is not None and v > 0]
self._initialize_processes(min(contenders))
play_context = PlayContext(new_play, self._options, self.passwords, self._connection_lockfile.fileno())

2
lib/ansible/playbook/play.py
View file

@ -87,7 +87,7 @@ class Play(Base, Taggable, Become):
_any_errors_fatal = FieldAttribute(isa='bool', default=False, always_post_validate=True)
_force_handlers = FieldAttribute(isa='bool', always_post_validate=True)
_max_fail_percentage = FieldAttribute(isa='percent', always_post_validate=True)
_serial = FieldAttribute(isa='string', always_post_validate=True)
_serial = FieldAttribute(isa='list', default=[], always_post_validate=True)
_strategy = FieldAttribute(isa='string', default='linear', always_post_validate=True)
# =================================================================================

34
lib/ansible/utils/helpers.py Normal file
View file

@ -0,0 +1,34 @@
# (c) 2016, Ansible by Red Hat <support@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six import string_types
def pct_to_int(value, num_items, min_value=1):
'''
Converts a given value to a percentage if specified as "x%",
otherwise converts the given value to an integer.
'''
if isinstance(value, string_types) and value.endswith('%'):
value_pct = int(value.replace("%",""))
return int((value_pct/100.0) * num_items) or min_value
else:
return int(value)

38
test/units/executor/test_playbook_executor.py
View file

@ -25,10 +25,11 @@ from ansible.compat.tests.mock import patch, MagicMock
from ansible.errors import AnsibleError, AnsibleParserError
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.playbook import Playbook
from ansible.template import Templar
from units.mock.loader import DictDataLoader
class TestPlayIterator(unittest.TestCase):
class TestPlaybookExecutor(unittest.TestCase):
def setUp(self):
pass
@ -58,6 +59,20 @@ class TestPlayIterator(unittest.TestCase):
tasks:
- debug: var=inventory_hostname
''',
'serial_list.yml': '''
- hosts: all
gather_facts: no
serial: [1, 2, 3]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_mixed.yml': '''
- hosts: all
gather_facts: no
serial: [1, "20%", -1]
tasks:
- debug: var=inventory_hostname
''',
})
mock_inventory = MagicMock()
@ -68,8 +83,10 @@ class TestPlayIterator(unittest.TestCase):
mock_options = MagicMock()
mock_options.syntax.value = True
templar = Templar(loader=fake_loader)
pbe = PlaybookExecutor(
playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml'],
playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml', 'serial_list.yml', 'serial_list_mixed.yml'],
inventory=mock_inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
@ -79,27 +96,44 @@ class TestPlayIterator(unittest.TestCase):
playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']])
playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
playbook = Playbook.load(pbe._playbooks[3], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1','host2'],['host3','host4','host5'],['host6','host7','host8'],['host9']])
playbook = Playbook.load(pbe._playbooks[4], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1','host2'],['host3','host4','host5','host6','host7','host8','host9']])
# Test when serial percent is under 1.0
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2']
self.assertEqual(pbe._get_serialized_batches(play), [['host0'],['host1'],['host2']])
# Test when there is a remainder for serial as a percent
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9','host10']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9'],['host10']])

31
test/units/utils/test_helpers.py Normal file
View file

@ -0,0 +1,31 @@
# (c) 2015, Marius Gedminas <marius@gedmin.as>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import unittest
from ansible.utils.helpers import pct_to_int
class TestHelpers(unittest.TestCase):
def test_pct_to_int(self):
self.assertEqual(pct_to_int(1, 100), 1)
self.assertEqual(pct_to_int(-1, 100), -1)
self.assertEqual(pct_to_int("1%", 10), 1)
self.assertEqual(pct_to_int("1%", 10, 0), 0)
self.assertEqual(pct_to_int("1", 100), 1)
self.assertEqual(pct_to_int("10%", 100), 10)