diff --git a/v2/ansible/executor/HostPlaybookIterator.py b/v2/ansible/executor/HostPlaybookIterator.py deleted file mode 100644 index 07fab067147..00000000000 --- a/v2/ansible/executor/HostPlaybookIterator.py +++ /dev/null @@ -1,36 +0,0 @@ -# (c) 2012-2014, Michael DeHaan -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -# Make coding more python3-ish -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -class HostPlaybookIterator: - - def __init__(self, host, playbook): - pass - - def get_next_task(self): - assert False - - def is_blocked(self): - # depending on strategy, either - # ‘linear’ -- all prev tasks must be completed for all hosts - # ‘free’ -- this host doesn’t have any more work to do - assert False - - diff --git a/v2/ansible/executor/TaskResult.py b/v2/ansible/executor/__init__.py similarity index 100% rename from v2/ansible/executor/TaskResult.py rename to v2/ansible/executor/__init__.py diff --git a/v2/ansible/executor/HostLog.py b/v2/ansible/executor/host_log.py similarity index 100% rename from v2/ansible/executor/HostLog.py rename to v2/ansible/executor/host_log.py diff --git a/v2/ansible/executor/HostLogManager.py b/v2/ansible/executor/host_log_manager.py similarity index 100% rename from v2/ansible/executor/HostLogManager.py rename to v2/ansible/executor/host_log_manager.py diff --git a/v2/ansible/executor/PlaybookExecutor.py b/v2/ansible/executor/playbook_executor.py similarity index 100% rename from v2/ansible/executor/PlaybookExecutor.py rename to v2/ansible/executor/playbook_executor.py diff --git a/v2/ansible/executor/playbook_iterator.py b/v2/ansible/executor/playbook_iterator.py new file mode 100644 index 00000000000..0d4f09b1e4a --- /dev/null +++ b/v2/ansible/executor/playbook_iterator.py @@ -0,0 +1,97 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +class PlaybookState: + + ''' + A helper class, which keeps track of the task iteration + state for a given playbook. This is used in the PlaybookIterator + class on a per-host basis. + ''' + def __init__(self, parent_iterator): + self._parent_iterator = parent_iterator + self._cur_play = 0 + self._task_list = None + self._cur_task_pos = 0 + + def next(self): + ''' + Determines and returns the next available task from the playbook, + advancing through the list of plays as it goes. + ''' + + while True: + # when we hit the end of the playbook entries list, we return + # None to indicate we're there + if self._cur_play > len(self._parent_iterator._playbook._entries) - 1: + return None + + # initialize the task list by calling the .compile() method + # on the play, which will call compile() for all child objects + if self._task_list is None: + self._task_list = self._parent_iterator._playbook._entries[self._cur_play].compile() + + # if we've hit the end of this plays task list, move on to the next + # and reset the position values for the next iteration + if self._cur_task_pos > len(self._task_list) - 1: + self._cur_play += 1 + self._task_list = None + self._cur_task_pos = 0 + continue + else: + # FIXME: do tag/conditional evaluation here and advance + # the task position if it should be skipped without + # returning a task + task = self._task_list[self._cur_task_pos] + self._cur_task_pos += 1 + + # Skip the task if it is the member of a role which has already + # been run, unless the role allows multiple executions + if task._role: + # FIXME: this should all be done via member functions + # instead of direct access to internal variables + if task._role.has_run() and not task._role._metadata._allow_duplicates: + continue + + return task + +class PlaybookIterator: + + ''' + The main iterator class, which keeps the state of the playbook + on a per-host basis using the above PlaybookState class. + ''' + + def __init__(self, inventory, log_manager, playbook): + self._playbook = playbook + self._log_manager = log_manager + self._host_entries = dict() + + # build the per-host dictionary of playbook states + for host in inventory.get_hosts(): + self._host_entries[host.get_name()] = PlaybookState(parent_iterator=self) + + def get_next_task_for_host(self, host): + ''' fetch the next task for the given host ''' + if host.get_name() not in self._host_entries: + raise AnsibleError("invalid host specified for playbook iteration") + + return self._host_entries[host.get_name()].next() diff --git a/v2/ansible/executor/TaskExecutor.py b/v2/ansible/executor/task_executor.py similarity index 100% rename from v2/ansible/executor/TaskExecutor.py rename to v2/ansible/executor/task_executor.py diff --git a/v2/ansible/executor/TaskQueueManager.py b/v2/ansible/executor/task_queue_manager.py similarity index 100% rename from v2/ansible/executor/TaskQueueManager.py rename to v2/ansible/executor/task_queue_manager.py diff --git a/v2/ansible/executor/TemplateEngine.py b/v2/ansible/executor/task_result.py similarity index 100% rename from v2/ansible/executor/TemplateEngine.py rename to v2/ansible/executor/task_result.py diff --git a/v2/ansible/executor/VariableCache.py b/v2/ansible/executor/template_engine.py similarity index 100% rename from v2/ansible/executor/VariableCache.py rename to v2/ansible/executor/template_engine.py diff --git a/v2/ansible/parsing/yaml/__init__.py b/v2/ansible/parsing/yaml/__init__.py index 4273abee539..a6c63feaa70 100644 --- a/v2/ansible/parsing/yaml/__init__.py +++ b/v2/ansible/parsing/yaml/__init__.py @@ -148,6 +148,10 @@ class DataLoader(): raise AnsibleParserError(YAML_SYNTAX_ERROR, obj=err_obj, show_content=show_content) + def get_basedir(self): + ''' returns the current basedir ''' + return self._basedir + def set_basedir(self, basedir): ''' sets the base directory, used to find files when a relative path is given ''' diff --git a/v2/ansible/playbook/__init__.py b/v2/ansible/playbook/__init__.py index f8f42b1163d..2d594c4802e 100644 --- a/v2/ansible/playbook/__init__.py +++ b/v2/ansible/playbook/__init__.py @@ -57,6 +57,9 @@ class Playbook: basedir = os.path.dirname(file_name) self._loader.set_basedir(basedir) + # also add the basedir to the list of module directories + push_basedir(basedir) + ds = self._loader.load_from_file(file_name) if not isinstance(ds, list): raise AnsibleParserError("playbooks must be a list of plays", obj=ds) @@ -75,4 +78,5 @@ class Playbook: self._entries.append(entry_obj) - + def get_entries(self): + return self._entries[:] diff --git a/v2/ansible/playbook/block.py b/v2/ansible/playbook/block.py index a082e97e5eb..0fc19113f05 100644 --- a/v2/ansible/playbook/block.py +++ b/v2/ansible/playbook/block.py @@ -22,6 +22,7 @@ __metaclass__ = type from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base from ansible.playbook.helpers import load_list_of_tasks +from ansible.playbook.task_include import TaskInclude class Block(Base): @@ -35,8 +36,10 @@ class Block(Base): # similar to the 'else' clause for exceptions #_otherwise = FieldAttribute(isa='list') - def __init__(self, role=None): - self.role = role + def __init__(self, parent_block=None, role=None, task_include=None): + self._parent_block = parent_block + self._role = role + self._task_include = task_include super(Block, self).__init__() def get_variables(self): @@ -45,8 +48,8 @@ class Block(Base): return dict() @staticmethod - def load(data, role=None, loader=None): - b = Block(role=role) + def load(data, parent_block=None, role=None, task_include=None, loader=None): + b = Block(parent_block=parent_block, role=role, task_include=task_include) return b.load_data(data, loader=loader) def munge(self, ds): @@ -79,3 +82,14 @@ class Block(Base): #def _load_otherwise(self, attr, ds): # return self._load_list_of_tasks(ds, block=self, loader=self._loader) + def compile(self): + ''' + Returns the task list for this object + ''' + + task_list = [] + for task in self.block: + # FIXME: evaulate task tags/conditionals here + task_list.extend(task.compile()) + + return task_list diff --git a/v2/ansible/playbook/helpers.py b/v2/ansible/playbook/helpers.py index f692f4baf6c..1d79721dce2 100644 --- a/v2/ansible/playbook/helpers.py +++ b/v2/ansible/playbook/helpers.py @@ -15,11 +15,16 @@ # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . + +import os + from types import NoneType from ansible.errors import AnsibleParserError +from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject -def load_list_of_blocks(ds, role=None, loader=None): + +def load_list_of_blocks(ds, parent_block=None, role=None, task_include=None, loader=None): ''' Given a list of mixed task/block data (parsed from YAML), return a list of Block() objects, where implicit blocks @@ -34,7 +39,7 @@ def load_list_of_blocks(ds, role=None, loader=None): block_list = [] if ds: for block in ds: - b = Block.load(block, role=role, loader=loader) + b = Block.load(block, parent_block=parent_block, role=role, task_include=task_include, loader=loader) block_list.append(b) return block_list @@ -58,7 +63,17 @@ def load_list_of_tasks(ds, block=None, role=None, task_include=None, loader=None raise AnsibleParserError("task/handler entries must be dictionaries (got a %s)" % type(task), obj=ds) if 'include' in task: + cur_basedir = None + if isinstance(task, AnsibleBaseYAMLObject) and loader: + pos_info = task.get_position_info() + new_basedir = os.path.dirname(pos_info[0]) + cur_basedir = loader.get_basedir() + loader.set_basedir(new_basedir) + t = TaskInclude.load(task, block=block, role=role, task_include=task_include, loader=loader) + + if cur_basedir and loader: + loader.set_basedir(cur_basedir) else: t = Task.load(task, block=block, role=role, task_include=task_include, loader=loader) @@ -85,3 +100,15 @@ def load_list_of_roles(ds, loader=None): return roles +def compile_block_list(block_list): + ''' + Given a list of blocks, compile them into a flat list of tasks + ''' + + task_list = [] + + for block in block_list: + task_list.extend(block.compile()) + + return task_list + diff --git a/v2/ansible/playbook/play.py b/v2/ansible/playbook/play.py index 07ee4707b40..c3d11e6cb22 100644 --- a/v2/ansible/playbook/play.py +++ b/v2/ansible/playbook/play.py @@ -25,7 +25,8 @@ from ansible.parsing.yaml import DataLoader from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base -from ansible.playbook.helpers import load_list_of_blocks, load_list_of_roles +from ansible.playbook.helpers import load_list_of_blocks, load_list_of_roles, compile_block_list +from ansible.playbook.role import Role __all__ = ['Play'] @@ -155,3 +156,41 @@ class Play(Base): return load_list_of_roles(ds, loader=self._loader) # FIXME: post_validation needs to ensure that su/sudo are not both set + + def _compile_roles(self): + ''' + Handles the role compilation step, returning a flat list of tasks + with the lowest level dependencies first. For example, if a role R + has a dependency D1, which also has a dependency D2, the tasks from + D2 are merged first, followed by D1, and lastly by the tasks from + the parent role R last. This is done for all roles in the Play. + ''' + + task_list = [] + + if len(self.roles) > 0: + for ri in self.roles: + # The internal list of roles are actualy RoleInclude objects, + # so we load the role from that now + role = Role.load(ri) + + # FIXME: evauluate conditional of roles here? + task_list.extend(role.compile()) + + return task_list + + def compile(self): + ''' + Compiles and returns the task list for this play, compiled from the + roles (which are themselves compiled recursively) and/or the list of + tasks specified in the play. + ''' + + task_list = [] + + task_list.extend(compile_block_list(self.pre_tasks)) + task_list.extend(self._compile_roles()) + task_list.extend(compile_block_list(self.tasks)) + task_list.extend(compile_block_list(self.post_tasks)) + + return task_list diff --git a/v2/ansible/playbook/role/__init__.py b/v2/ansible/playbook/role/__init__.py index 67485f0f9c2..ab8a779fdef 100644 --- a/v2/ansible/playbook/role/__init__.py +++ b/v2/ansible/playbook/role/__init__.py @@ -30,7 +30,7 @@ from ansible.errors import AnsibleError, AnsibleParserError from ansible.parsing.yaml import DataLoader from ansible.playbook.attribute import FieldAttribute from ansible.playbook.base import Base -from ansible.playbook.helpers import load_list_of_blocks +from ansible.playbook.helpers import load_list_of_blocks, compile_block_list from ansible.playbook.role.include import RoleInclude from ansible.playbook.role.metadata import RoleMetadata @@ -87,6 +87,10 @@ class Role: if parent_role: self.add_parent(parent_role) + # save the current base directory for the loader and set it to the current role path + cur_basedir = self._loader.get_basedir() + self._loader.set_basedir(self._role_path) + # load the role's files, if they exist metadata = self._load_role_yaml('meta') if metadata: @@ -110,6 +114,9 @@ class Role: if not isinstance(self._default_vars, (dict, NoneType)): raise AnsibleParserError("The default/main.yml file for role '%s' must contain a dictionary of variables" % self._role_name, obj=ds) + # and finally restore the previous base directory + self._loader.set_basedir(cur_basedir) + def _load_role_yaml(self, subdir): file_path = os.path.join(self._role_path, subdir) if self._loader.path_exists(file_path) and self._loader.is_directory(file_path): @@ -186,3 +193,26 @@ class Role: return direct_deps + child_deps + def get_task_blocks(self): + return self._task_blocks[:] + + def get_handler_blocks(self): + return self._handler_blocks[:] + + def compile(self): + ''' + Returns the task list for this role, which is created by first + recursively compiling the tasks for all direct dependencies, and + then adding on the tasks for this role. + ''' + + task_list = [] + + deps = self.get_direct_dependencies() + for dep in deps: + task_list.extend(dep.compile()) + + task_list.extend(compile_block_list(self._task_blocks)) + + return task_list + diff --git a/v2/ansible/playbook/role/definition.py b/v2/ansible/playbook/role/definition.py index 08d62afbe4b..34b0248820c 100644 --- a/v2/ansible/playbook/role/definition.py +++ b/v2/ansible/playbook/role/definition.py @@ -124,6 +124,7 @@ class RoleDefinition(Base): # FIXME: make the parser smart about list/string entries # in the yaml so the error line/file can be reported # here + raise AnsibleError("the role '%s' was not found" % role_name) def _split_role_params(self, ds): diff --git a/v2/ansible/playbook/task.py b/v2/ansible/playbook/task.py index 95571819af3..c4c22025ed0 100644 --- a/v2/ansible/playbook/task.py +++ b/v2/ansible/playbook/task.py @@ -60,6 +60,7 @@ class Task(Base): _delay = FieldAttribute(isa='int') _delegate_to = FieldAttribute(isa='string') _environment = FieldAttribute(isa='dict') + _failed_when = FieldAttribute(isa='string') _first_available_file = FieldAttribute(isa='list') _ignore_errors = FieldAttribute(isa='bool') @@ -179,3 +180,11 @@ class Task(Base): return new_ds + def compile(self): + ''' + For tasks, this is just a dummy method returning an array + with 'self' in it, so we don't have to care about task types + further up the chain. + ''' + + return [self] diff --git a/v2/ansible/playbook/task_include.py b/v2/ansible/playbook/task_include.py index 798ce020d1c..dbbc388f688 100644 --- a/v2/ansible/playbook/task_include.py +++ b/v2/ansible/playbook/task_include.py @@ -24,7 +24,7 @@ from ansible.parsing.splitter import split_args, parse_kv from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping from ansible.playbook.attribute import Attribute, FieldAttribute from ansible.playbook.base import Base -from ansible.playbook.helpers import load_list_of_tasks +from ansible.playbook.helpers import load_list_of_blocks, compile_block_list from ansible.plugins import lookup_finder @@ -57,11 +57,12 @@ class TaskInclude(Base): _when = FieldAttribute(isa='list', default=[]) def __init__(self, block=None, role=None, task_include=None): - self._tasks = [] self._block = block self._role = role self._task_include = task_include + self._task_blocks = [] + super(TaskInclude, self).__init__() @staticmethod @@ -136,11 +137,27 @@ class TaskInclude(Base): def _load_include(self, attr, ds): - ''' loads the file name specified in the ds and returns a list of tasks ''' + ''' loads the file name specified in the ds and returns a list of blocks ''' data = self._loader.load_from_file(ds) if not isinstance(data, list): raise AnsibleParsingError("included task files must contain a list of tasks", obj=ds) - self._tasks = load_list_of_tasks(data, task_include=self, loader=self._loader) + self._task_blocks = load_list_of_blocks( + data, + parent_block=self._block, + task_include=self, + role=self._role, + loader=self._loader + ) return ds + + def compile(self): + ''' + Returns the task list for the included tasks. + ''' + + task_list = [] + task_list.extend(compile_block_list(self._task_blocks)) + return task_list + diff --git a/v2/ansible/executor/VariableManager.py b/v2/test/executor/__init__.py similarity index 100% rename from v2/ansible/executor/VariableManager.py rename to v2/test/executor/__init__.py diff --git a/v2/test/executor/test_playbook_iterator.py b/v2/test/executor/test_playbook_iterator.py new file mode 100644 index 00000000000..96db014fd6f --- /dev/null +++ b/v2/test/executor/test_playbook_iterator.py @@ -0,0 +1,83 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.compat.tests import unittest +from ansible.compat.tests.mock import patch, MagicMock + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.executor.playbook_iterator import PlaybookIterator +from ansible.playbook import Playbook + +from test.mock.loader import DictDataLoader + +class TestPlaybookIterator(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_playbook_iterator(self): + fake_loader = DictDataLoader({ + "test_play.yml": """ + - hosts: all + roles: + - test_role + pre_tasks: + - debug: msg="this is a pre_task" + tasks: + - debug: msg="this is a regular task" + post_tasks: + - debug: msg="this is a post_task" + """, + '/etc/ansible/roles/test_role/tasks/main.yml': """ + - debug: msg="this is a role task" + """, + }) + + p = Playbook.load('test_play.yml', loader=fake_loader) + + hosts = [] + for i in range(0, 10): + host = MagicMock() + host.get_name.return_value = 'host%02d' % i + hosts.append(host) + + inventory = MagicMock() + inventory.get_hosts.return_value = hosts + + itr = PlaybookIterator(inventory, None, p) + task = itr.get_next_task_for_host(hosts[0]) + print(task) + self.assertIsNotNone(task) + task = itr.get_next_task_for_host(hosts[0]) + print(task) + self.assertIsNotNone(task) + task = itr.get_next_task_for_host(hosts[0]) + print(task) + self.assertIsNotNone(task) + task = itr.get_next_task_for_host(hosts[0]) + print(task) + self.assertIsNotNone(task) + task = itr.get_next_task_for_host(hosts[0]) + print(task) + self.assertIsNone(task) diff --git a/v2/test/playbook/test_block.py b/v2/test/playbook/test_block.py index 348681527bb..9c1d06cbcb8 100644 --- a/v2/test/playbook/test_block.py +++ b/v2/test/playbook/test_block.py @@ -75,3 +75,9 @@ class TestBlock(unittest.TestCase): self.assertEqual(len(b.block), 1) assert isinstance(b.block[0], Task) + def test_block_compile(self): + ds = [dict(action='foo')] + b = Block.load(ds) + tasks = b.compile() + self.assertEqual(len(tasks), 1) + self.assertIsInstance(tasks[0], Task) diff --git a/v2/test/playbook/test_play.py b/v2/test/playbook/test_play.py index 14732a1f9fb..22486f41290 100644 --- a/v2/test/playbook/test_play.py +++ b/v2/test/playbook/test_play.py @@ -117,4 +117,16 @@ class TestPlay(unittest.TestCase): roles=['foo'], ), loader=fake_loader) + tasks = p.compile() + def test_play_compile(self): + p = Play.load(dict( + name="test play", + hosts=['foo'], + gather_facts=False, + tasks=[dict(action='shell echo "hello world"')], + )) + + tasks = p.compile() + self.assertEqual(len(tasks), 1) + self.assertIsInstance(tasks[0], Task) diff --git a/v2/test/playbook/test_playbook.py b/v2/test/playbook/test_playbook.py index 640057820e8..f3ba6785f3f 100644 --- a/v2/test/playbook/test_playbook.py +++ b/v2/test/playbook/test_playbook.py @@ -45,6 +45,7 @@ class TestPlaybook(unittest.TestCase): """, }) p = Playbook.load("test_file.yml", loader=fake_loader) + entries = p.get_entries() def test_bad_playbook_files(self): fake_loader = DictDataLoader({ diff --git a/v2/test/playbook/test_task_include.py b/v2/test/playbook/test_task_include.py index 42a63b72049..55f7461f050 100644 --- a/v2/test/playbook/test_task_include.py +++ b/v2/test/playbook/test_task_include.py @@ -45,6 +45,7 @@ class TestTaskInclude(unittest.TestCase): def test_basic_task_include(self): ti = TaskInclude.load(AnsibleMapping(include='foo.yml'), loader=self._fake_loader) + tasks = ti.compile() def test_task_include_with_loop(self): ti = TaskInclude.load(AnsibleMapping(include='foo.yml', with_items=['a', 'b', 'c']), loader=self._fake_loader)