From d84df2405dc84c1af5d41ddf9c0c2b1d499026f4 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Sat, 16 Sep 2017 23:32:34 -0400 Subject: [PATCH] move from with_: to loop: - old functionality is still available direct lookup use, the following are equivalent with_nested: [[1,2,3], ['a','b','c']] loop: "{{lookup('nested', [1,2,3], ['a','b','c'])}}" - avoid squashing with 'loop:' - fixed test to use new intenal attributes - removed most of 'lookup docs' as these now reside in the plugins --- CHANGELOG.md | 4 + docs/bin/dump_keywords.py | 5 +- docs/docsite/keyword_desc.yml | 1 + docs/docsite/rst/faq.rst | 2 +- docs/docsite/rst/glossary.rst | 20 +- docs/docsite/rst/guide_aws.rst | 2 +- docs/docsite/rst/guide_cloudstack.rst | 10 +- docs/docsite/rst/guide_gce.rst | 4 +- docs/docsite/rst/guide_packet.rst | 2 +- docs/docsite/rst/guide_rax.rst | 8 +- docs/docsite/rst/guide_rolling_upgrade.rst | 10 +- docs/docsite/rst/playbooks_async.rst | 6 +- docs/docsite/rst/playbooks_blocks.rst | 2 +- docs/docsite/rst/playbooks_conditionals.rst | 32 +- docs/docsite/rst/playbooks_delegation.rst | 4 +- docs/docsite/rst/playbooks_filters.rst | 14 +- docs/docsite/rst/playbooks_lookups.rst | 576 +---------------- docs/docsite/rst/playbooks_loops.rst | 607 ++---------------- docs/docsite/rst/playbooks_python_version.rst | 8 +- docs/docsite/rst/playbooks_tags.rst | 2 +- lib/ansible/executor/task_executor.py | 22 +- lib/ansible/playbook/task.py | 21 +- lib/ansible/vars/manager.py | 2 +- test/units/executor/test_task_executor.py | 4 +- 24 files changed, 157 insertions(+), 1211 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6aa374fec9..412ca090992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,12 @@ Ansible Changes By Release ### Major Changes * Removed the previously deprecated 'accelerate' mode and all associated keywords and code. +* Removed the previouslly deprecated 'accelerate' mode and all associated keywords and code. +* New simpler and more intuitive 'loop' keyword for task loops ### Deprecations +* previouslly deprecated 'hostfile' config settings have been 're-deprecated' as previouslly code did not warn about deprecated configuration settings. +* The ``with_`` loops are deprecated in favor of the new ``loop`` keyword #### Deprecated Modules (to be removed in 2.9): diff --git a/docs/bin/dump_keywords.py b/docs/bin/dump_keywords.py index d29e1ee4b60..38b46147baf 100755 --- a/docs/bin/dump_keywords.py +++ b/docs/bin/dump_keywords.py @@ -49,14 +49,15 @@ for aclass in class_list: # loop is really with_ for users if name == 'Task': - oblist[name]['with_'] = 'with_ is how loops are defined, it can use any available lookup plugin to generate the item list' + oblist[name]['with_'] = 'DEPRECATED: use ``loop`` instead, with_ used to be how loops were defined, ' + 'it can use any available lookup plugin to generate the item list' # local_action is implicit with action if 'action' in oblist[name]: oblist[name]['local_action'] = 'Same as action but also implies ``delegate_to: localhost``' # remove unusable (used to be private?) - for nouse in ('loop', 'loop_args'): + for nouse in ('loop_args'): if nouse in oblist[name]: del oblist[name][nouse] diff --git a/docs/docsite/keyword_desc.yml b/docs/docsite/keyword_desc.yml index d7356d8dc38..b9f30b7b2e4 100644 --- a/docs/docsite/keyword_desc.yml +++ b/docs/docsite/keyword_desc.yml @@ -29,6 +29,7 @@ gather_timeout: Allows you to set the timeout for the fact gathering plugin cont handlers: "A section with tasks that are treated as handlers, these won't get executed normally, only when notified. After each section of tasks is complete." hosts: "A list of groups, hosts or host pattern that translates into a list of hosts that are the play's target." ignore_errors: Boolean that allows you to ignore task failures and continue with play. It does not affect connection errors. +loop: "Takes a list for the task to iterate over, saving each list element into the ``item`` variable (configurable via loop_control)" loop_control: "Several keys here allow you to modify/set loop behaviour in a task see http://docs.ansible.com/ansible/latest/playbooks_loops.html#loop-control for details." max_fail_percentage: can be used to abort the run after a given percentage of hosts in the current batch has failed. name: "It's a name, works mostly for documentation, in the case of tasks/handlers it can be an identifier." diff --git a/docs/docsite/rst/faq.rst b/docs/docsite/rst/faq.rst index db3cc24fe70..fcf4883161c 100644 --- a/docs/docsite/rst/faq.rst +++ b/docs/docsite/rst/faq.rst @@ -378,7 +378,7 @@ A steadfast rule is 'always use {{ }} except when `when:`'. Conditionals are always run through Jinja2 as to resolve the expression, so `when:`, `failed_when:` and `changed_when:` are always templated and you should avoid adding `{{}}`. -In most other cases you should always use the brackets, even if previously you could use variables without specifying (like `with_` clauses), +In most other cases you should always use the brackets, even if previously you could use variables without specifying (like `loop` or `with_` clauses), as this made it hard to distinguish between an undefined variable and a string. Another rule is 'moustaches don't stack'. We often see this: diff --git a/docs/docsite/rst/glossary.rst b/docs/docsite/rst/glossary.rst index b507f3e1eaf..cce42f5a947 100644 --- a/docs/docsite/rst/glossary.rst +++ b/docs/docsite/rst/glossary.rst @@ -255,22 +255,22 @@ when a term comes up on the mailing list. that we are managing the local host and not a remote machine. Lookup Plugin - A lookup plugin is a way to get data into Ansible from the outside - world. These are how such things as ``with_items``, a basic looping - plugin, are implemented. There are also lookup plugins like - ``with_file`` which load data from a file and ones for querying - environment variables, DNS text records, or key value stores. Lookup - plugins can also be accessed in templates, e.g., + A lookup plugin is a way to get data into Ansible from the outside world. + Lookup plugins are an extension of Jinja2 and can be accessed in templates, e.g., ``{{ lookup('file','/path/to/file') }}``. + These are how such things as ``with_items``, are implemented. + There are also lookup plugins like ``file`` which loads data from + a file and ones for querying environment variables, DNS text records, + or key value stores. Loops Generally, Ansible is not a programming language. It prefers to be - more declarative, though various constructs like ``with_items`` allow + more declarative, though various constructs like ``loop`` allow a particular task to be repeated for multiple items in a list. - Certain modules, like :ref:`yum ` and :ref:`apt `, are actually - optimized for this, and can install all packages given in those lists + Certain modules, like :ref:`yum ` and :ref:`apt `, actually take + lists directly, and can install all packages given in those lists within a single transaction, dramatically speeding up total time to - configuration. + configuration, so they can be used without loops. Modules Modules are the units of work that Ansible ships out to remote diff --git a/docs/docsite/rst/guide_aws.rst b/docs/docsite/rst/guide_aws.rst index 7135002dca2..7812ce82401 100644 --- a/docs/docsite/rst/guide_aws.rst +++ b/docs/docsite/rst/guide_aws.rst @@ -115,7 +115,7 @@ From this, we'll use the add_host module to dynamically create a host group cons - name: Add all instance public IPs to host group add_host: hostname={{ item.public_ip }} groups=ec2hosts - with_items: "{{ ec2.instances }}" + loop: "{{ ec2.instances }}" With the host group now created, a second play at the bottom of the same provisioning playbook file might now have some configuration steps:: diff --git a/docs/docsite/rst/guide_cloudstack.rst b/docs/docsite/rst/guide_cloudstack.rst index 4bfefddeb23..4a650c220dd 100644 --- a/docs/docsite/rst/guide_cloudstack.rst +++ b/docs/docsite/rst/guide_cloudstack.rst @@ -125,7 +125,7 @@ Or by looping over a regions list if you want to do the task in every region: name: my-ssh-key public_key: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" api_region: "{{ item }}" - with_items: + loop: - exoscale - exmaple_cloud_one - exmaple_cloud_two @@ -255,7 +255,7 @@ Now to the fun part. We create a playbook to create our infrastructure we call i ip_address: "{{ public_ip }}" port: "{{ item.port }}" cidr: "{{ item.cidr | default('0.0.0.0/0') }}" - with_items: "{{ cs_firewall }}" + loop: "{{ cs_firewall }}" when: public_ip is defined - name: ensure static NATs @@ -326,7 +326,7 @@ The playbook looks like the following: - name: ensure security groups exist cs_securitygroup: name: "{{ item }}" - with_items: + loop: - default - web @@ -335,7 +335,7 @@ The playbook looks like the following: security_group: default start_port: "{{ item }}" end_port: "{{ item }}" - with_items: + loop: - 22 - name: add inbound TCP rules to security group web @@ -343,7 +343,7 @@ The playbook looks like the following: security_group: web start_port: "{{ item }}" end_port: "{{ item }}" - with_items: + loop: - 80 - 443 diff --git a/docs/docsite/rst/guide_gce.rst b/docs/docsite/rst/guide_gce.rst index fda21c2946e..83d860c0831 100644 --- a/docs/docsite/rst/guide_gce.rst +++ b/docs/docsite/rst/guide_gce.rst @@ -213,11 +213,11 @@ A playbook would looks like this: - name: Wait for SSH to come up wait_for: host={{ item.public_ip }} port=22 delay=10 timeout=60 - with_items: "{{ gce.instance_data }}" + loop: "{{ gce.instance_data }}" - name: Add host to groupname add_host: hostname={{ item.public_ip }} groupname=new_instances - with_items: "{{ gce.instance_data }}" + loop: "{{ gce.instance_data }}" - name: Manage new instances hosts: new_instances diff --git a/docs/docsite/rst/guide_packet.rst b/docs/docsite/rst/guide_packet.rst index 5f19775d376..a61234d21b3 100644 --- a/docs/docsite/rst/guide_packet.rst +++ b/docs/docsite/rst/guide_packet.rst @@ -178,7 +178,7 @@ The following playbook will create an SSH key, 3 Packet servers, and then wait u port: 22 state: started timeout: 500 - with_items: "{{ newhosts.devices }}" + loop: "{{ newhosts.devices }}" As with most Ansible modules, the default states of the Packet modules are idempotent, meaning the resources in your project will remain the same after re-runs of a playbook. Thus, we can keep the ``packet_sshkey`` module call in our playbook. If the public key is already in your Packet account, the call will have no effect. diff --git a/docs/docsite/rst/guide_rax.rst b/docs/docsite/rst/guide_rax.rst index 081eb32a2e0..f118ec10ea2 100644 --- a/docs/docsite/rst/guide_rax.rst +++ b/docs/docsite/rst/guide_rax.rst @@ -134,7 +134,7 @@ The rax module returns data about the nodes it creates, like IP addresses, hostn ansible_host: "{{ item.rax_accessipv4 }}" ansible_ssh_pass: "{{ item.rax_adminpass }}" groups: raxhosts - with_items: "{{ rax.success }}" + loop: "{{ rax.success }}" when: rax.action == 'create' With the host group now created, the next play in this playbook could now configure servers belonging to the raxhosts group. @@ -522,7 +522,7 @@ Build a complete webserver environment with servers, custom networks and load ba ansible_ssh_pass: "{{ item.rax_adminpass }}" ansible_user: root groups: web - with_items: "{{ rax.success }}" + loop: "{{ rax.success }}" when: rax.action == 'create' - name: Add servers to Load balancer @@ -536,7 +536,7 @@ Build a complete webserver environment with servers, custom networks and load ba type: primary wait: yes region: IAD - with_items: "{{ rax.success }}" + loop: "{{ rax.success }}" when: rax.action == 'create' - name: Configure servers @@ -608,7 +608,7 @@ Using a Control Machine ansible_user: root rax_id: "{{ item.rax_id }}" groups: web,new_web - with_items: "{{ rax.success }}" + loop: "{{ rax.success }}" when: rax.action == 'create' - name: Wait for rackconnect and managed cloud automation to complete diff --git a/docs/docsite/rst/guide_rolling_upgrade.rst b/docs/docsite/rst/guide_rolling_upgrade.rst index e23445f94ab..9666a370dc7 100644 --- a/docs/docsite/rst/guide_rolling_upgrade.rst +++ b/docs/docsite/rst/guide_rolling_upgrade.rst @@ -213,16 +213,16 @@ Here is the next part of the update play:: - name: disable nagios alerts for this host webserver service nagios: action=disable_alerts host={{ inventory_hostname }} services=webserver delegate_to: "{{ item }}" - with_items: "{{ groups.monitoring }}" + loop: "{{ groups.monitoring }}" - name: disable the server in haproxy shell: echo "disable server myapplb/{{ inventory_hostname }}" | socat stdio /var/lib/haproxy/stats delegate_to: "{{ item }}" - with_items: "{{ groups.lbservers }}" + loop: "{{ groups.lbservers }}" The ``pre_tasks`` keyword just lets you list tasks to run before the roles are called. This will make more sense in a minute. If you look at the names of these tasks, you can see that we are disabling Nagios alerts and then removing the webserver that we are currently updating from the HAProxy load balancing pool. -The ``delegate_to`` and ``with_items`` arguments, used together, cause Ansible to loop over each monitoring server and load balancer, and perform that operation (delegate that operation) on the monitoring or load balancing server, "on behalf" of the webserver. In programming terms, the outer loop is the list of web servers, and the inner loop is the list of monitoring servers. +The ``delegate_to`` and ``loop`` arguments, used together, cause Ansible to loop over each monitoring server and load balancer, and perform that operation (delegate that operation) on the monitoring or load balancing server, "on behalf" of the webserver. In programming terms, the outer loop is the list of web servers, and the inner loop is the list of monitoring servers. Note that the HAProxy step looks a little complicated. We're using HAProxy in this example because it's freely available, though if you have (for instance) an F5 or Netscaler in your infrastructure (or maybe you have an AWS Elastic IP setup?), you can use modules included in core Ansible to communicate with them instead. You might also wish to use other monitoring modules instead of nagios, but this just shows the main goal of the 'pre tasks' section -- take the server out of monitoring, and take it out of rotation. @@ -239,12 +239,12 @@ Finally, in the ``post_tasks`` section, we reverse the changes to the Nagios con - name: Enable the server in haproxy shell: echo "enable server myapplb/{{ inventory_hostname }}" | socat stdio /var/lib/haproxy/stats delegate_to: "{{ item }}" - with_items: "{{ groups.lbservers }}" + loop: "{{ groups.lbservers }}" - name: re-enable nagios alerts nagios: action=enable_alerts host={{ inventory_hostname }} services=webserver delegate_to: "{{ item }}" - with_items: "{{ groups.monitoring }}" + loop: "{{ groups.monitoring }}" Again, if you were using a Netscaler or F5 or Elastic Load Balancer, you would just substitute in the appropriate modules instead. diff --git a/docs/docsite/rst/playbooks_async.rst b/docs/docsite/rst/playbooks_async.rst index 026c006e871..6b9f62378ce 100644 --- a/docs/docsite/rst/playbooks_async.rst +++ b/docs/docsite/rst/playbooks_async.rst @@ -95,7 +95,7 @@ of tasks running concurrently, you can do it this way:: - 5 durations: "{{ item }}" include_tasks: execute_batch.yml - with_items: + loop: - "{{ sleep_durations | batch(2) | list }}" ##################### @@ -105,7 +105,7 @@ of tasks running concurrently, you can do it this way:: command: sleep {{ async_item }} async: 45 poll: 0 - with_items: "{{ durations }}" + loop: "{{ durations }}" loop_control: loop_var: "async_item" register: async_results @@ -113,7 +113,7 @@ of tasks running concurrently, you can do it this way:: - name: Check sync status async_status: jid: "{{ async_result_item.ansible_job_id }}" - with_items: "{{ async_results.results }}" + loop: "{{ async_results.results }}" loop_control: loop_var: "async_result_item" register: async_poll_results diff --git a/docs/docsite/rst/playbooks_blocks.rst b/docs/docsite/rst/playbooks_blocks.rst index 3d64f568fba..1da95ea23b4 100644 --- a/docs/docsite/rst/playbooks_blocks.rst +++ b/docs/docsite/rst/playbooks_blocks.rst @@ -16,7 +16,7 @@ by the tasks enclosed by a block. i.e. a `when` will be applied to the tasks, no - name: Install Apache block: - yum: name={{ item }} state=installed - with_items: + loop: - httpd - memcached - template: src=templates/src.j2 dest=/etc/foo.conf diff --git a/docs/docsite/rst/playbooks_conditionals.rst b/docs/docsite/rst/playbooks_conditionals.rst index f79c82c06e6..dec421aa9e1 100644 --- a/docs/docsite/rst/playbooks_conditionals.rst +++ b/docs/docsite/rst/playbooks_conditionals.rst @@ -117,24 +117,24 @@ As the examples show, you don't need to use `{{ }}` to use variables inside cond Loops and Conditionals `````````````````````` -Combining `when` with `with_items` (see :doc:`playbooks_loops`), be aware that the `when` statement is processed separately for each item. This is by design:: +Combining `when` with loops (see :doc:`playbooks_loops`), be aware that the `when` statement is processed separately for each item. This is by design:: tasks: - command: echo {{ item }} - with_items: [ 0, 2, 4, 6, 8, 10 ] + loop: [ 0, 2, 4, 6, 8, 10 ] when: item > 5 If you need to skip the whole task depending on the loop variable being defined, used the `|default` filter to provide an empty iterator:: - command: echo {{ item }} - with_items: "{{ mylist|default([]) }}" + loop: "{{ mylist|default([]) }}" when: item > 5 -If using `with_dict` which does not take a list:: +If using a dict in a loop:: - command: echo {{ item.key }} - with_dict: "{{ mydict|default({}) }}" + loop: "{{ lookup('dict', mydict|default({})) }}" when: item.value > 5 .. _loading_in_custom_facts: @@ -259,13 +259,12 @@ The following example shows how to template out a configuration file that was ve - name: template a file template: src={{ item }} dest=/etc/myapp/foo.conf - with_first_found: - - files: - - {{ ansible_distribution }}.conf - - default.conf - paths: - - search_location_one/somedir/ - - /opt/other_location/somedir/ + loop: "{{lookup('first_found', { 'files': myfiles, 'paths': mypaths})}}" + vars: + myfiles: + - "{{ansible_distribution}}.conf" + - default.conf + mypaths: ['search_location_one/somedir/', '/opt/other_location/somedir/'] Register Variables `````````````````` @@ -288,14 +287,13 @@ The 'register' keyword decides what variable to save a result in. The resulting when: motd_contents.stdout.find('hi') != -1 As shown previously, the registered variable's string contents are accessible with the 'stdout' value. -The registered result can be used in the "with_items" of a task if it is converted into +The registered result can be used in the loop of a task if it is converted into a list (or already is a list) as shown below. "stdout_lines" is already available on the object as well though you could also call "home_dirs.stdout.split()" if you wanted, and could split by other fields:: - - name: registered variable usage as a with_items list + - name: registered variable usage as a loop list hosts: all - tasks: - name: retrieve the list of home directories @@ -304,8 +302,8 @@ fields:: - name: add home dirs to the backup spooler file: path=/mnt/bkspool/{{ item }} src=/home/{{ item }} state=link - with_items: "{{ home_dirs.stdout_lines }}" - # same as with_items: "{{ home_dirs.stdout.split() }}" + loop: "{{ home_dirs.stdout_lines }}" + # same as loop: "{{ home_dirs.stdout.split() }}" As shown previously, the registered variable's string contents are accessible with the 'stdout' value. You may check the registered variable's string contents for emptiness:: diff --git a/docs/docsite/rst/playbooks_delegation.rst b/docs/docsite/rst/playbooks_delegation.rst index dab773b7009..e4967dc56dd 100644 --- a/docs/docsite/rst/playbooks_delegation.rst +++ b/docs/docsite/rst/playbooks_delegation.rst @@ -175,7 +175,7 @@ In case you have to specify more arguments you can use the following syntax:: to: "{{ mail_recipient }}" body: "{{ mail_body }}" run_once: True - + The `ansible_host` variable (`ansible_ssh_host` in 1.x or specific to ssh/paramiko plugins) reflects the host a task is delegated to. .. _delegate_facts: @@ -195,7 +195,7 @@ In 2.0, the directive `delegate_facts` may be set to `True` to assign the task's setup: delegate_to: "{{item}}" delegate_facts: True - with_items: "{{groups['dbservers']}}" + loop: "{{groups['dbservers']}}" The above will gather facts for the machines in the dbservers group and assign the facts to those machines and not to app_servers. This way you can lookup `hostvars['dbhost1']['default_ipv4']['address']` even though dbservers were not part of the play, or left out by using `--limit`. diff --git a/docs/docsite/rst/playbooks_filters.rst b/docs/docsite/rst/playbooks_filters.rst index 30b7029f6f7..329c0fc75b6 100644 --- a/docs/docsite/rst/playbooks_filters.rst +++ b/docs/docsite/rst/playbooks_filters.rst @@ -80,7 +80,7 @@ As of Ansible 1.8, it is possible to use the default filter to omit module param - name: touch files with an optional mode file: dest={{item.path}} state=touch mode={{item.mode|default(omit)}} - with_items: + loop: - path: /tmp/foo - path: /tmp/bar - path: /tmp/baz @@ -234,7 +234,7 @@ JSON Query Filter .. versionadded:: 2.2 -Sometimes you end up with a complex data structure in JSON format and you need to extract only a small set of data within it. The **json_query** filter lets you query a complex JSON structure and iterate over it using a with_items structure. +Sometimes you end up with a complex data structure in JSON format and you need to extract only a small set of data within it. The **json_query** filter lets you query a complex JSON structure and iterate over it using a loop structure. .. note:: This filter is built upon **jmespath**, and you can use the same syntax. For examples, see `jmespath examples `_. @@ -268,19 +268,19 @@ To extract all clusters from this structure, you can use the following query:: - name: "Display all cluster names" debug: var=item - with_items: "{{domain_definition|json_query('domain.cluster[*].name')}}" + loop: "{{domain_definition|json_query('domain.cluster[*].name')}}" Same thing for all server names:: - name: "Display all server names" debug: var=item - with_items: "{{domain_definition|json_query('domain.server[*].name')}}" + loop: "{{domain_definition|json_query('domain.server[*].name')}}" This example shows ports from cluster1:: - name: "Display all server names from cluster1" debug: var=item - with_items: "{{domain_definition|json_query(server_name_cluster1_query)}}" + loop: "{{domain_definition|json_query(server_name_cluster1_query)}}" vars: server_name_cluster1_query: "domain.server[?cluster=='cluster1'].port" @@ -291,7 +291,7 @@ Or, alternatively:: - name: "Display all server names from cluster1" debug: var: item - with_items: "{{domain_definition|json_query('domain.server[?cluster=`cluster1`].port')}}" + loop: "{{domain_definition|json_query('domain.server[?cluster=`cluster1`].port')}}" .. note:: Here, quoting literals using backticks avoids escaping quotes and maintains readability. @@ -299,7 +299,7 @@ In this example, we get a hash map with all ports and names of a cluster:: - name: "Display all server ports and names from cluster1" debug: var=item - with_items: "{{domain_definition|json_query(server_name_cluster1_query)}}" + loop: "{{domain_definition|json_query(server_name_cluster1_query)}}" vars: server_name_cluster1_query: "domain.server[?cluster=='cluster2'].{name: name, port: port}" diff --git a/docs/docsite/rst/playbooks_lookups.rst b/docs/docsite/rst/playbooks_lookups.rst index c424bfea9e7..3dce6b2f496 100644 --- a/docs/docsite/rst/playbooks_lookups.rst +++ b/docs/docsite/rst/playbooks_lookups.rst @@ -1,7 +1,7 @@ Lookups ------- -Lookup plugins allow access to outside data sources. Like all templating, these plugins are evaluated on the Ansible control machine, and can include reading the filesystem as well as contacting external datastores and services. This data is then made available using the standard templating system in Ansible. +Lookup plugins allow access to outside data sources. Like all templating, these plugins are evaluated on the Ansible control machine, and can include reading the filesystem as well as contacting external datastores and services. This data is then made available using the standard templating system in Ansible. .. note:: - Lookups occur on the local computer, not on the remote computer. @@ -13,578 +13,28 @@ Lookup plugins allow access to outside data sources. Like all templating, these .. contents:: Topics -.. _getting_file_contents: +.. _lookups_and_loops: -Intro to Lookups: Getting File Contents -``````````````````````````````````````` +Lookups and loops +````````````````` -The file lookup is the most basic lookup type. +Various *lookup plugins* allow additional ways to iterate over data. +In :doc:`Loops ` you will learn how to use them to walk over collections of numerous types. +However, they can also be used to pull in data from remote sources, such as shell commands or even key value stores. +Before Ansible 2.5, lookups were mostly used indirectly in ``with_] [section=section] [file=file.ini] [re=true] [default=]') - -The first value in the argument is the ``key``, which must be an entry that -appears exactly once on keys. All other arguments are optional. - - -========== ============ ========================================================================================= -Field Default Description ----------- ------------ ----------------------------------------------------------------------------------------- -type ini Type of the file. Can be ini or properties (for java properties). -file ansible.ini Name of the file to load -section global Default section where to lookup for key. -re False The key is a regexp. -encoding utf-8 Text encoding to use. -default empty string return value if the key is not in the ini file -========== ============ ========================================================================================= - -.. note:: In java properties files, there's no need to specify a section. - -.. _credstash_lookup: - -The Credstash Lookup -```````````````````` -.. versionadded:: 2.0 - -Credstash is a small utility for managing secrets using AWS's KMS and DynamoDB: https://github.com/fugue/credstash - -First, you need to store your secrets with credstash: - -.. code-block:: shell-session - - credstash put my-github-password secure123 - - # my-github-password has been stored - - -Example usage:: - - - --- - - name: "Test credstash lookup plugin -- get my github password" - debug: msg="Credstash lookup! {{ lookup('credstash', 'my-github-password') }}" - - -You can specify regions or tables to fetch secrets from:: - - - --- - - name: "Test credstash lookup plugin -- get my other password from us-west-1" - debug: msg="Credstash lookup! {{ lookup('credstash', 'my-other-password', region='us-west-1') }}" - - - - name: "Test credstash lookup plugin -- get the company's github password" - debug: msg="Credstash lookup! {{ lookup('credstash', 'company-github-password', table='company-passwords') }}" - - -If you use the context feature when putting your secret, you can get it by passing a dictionary to the context option like this:: - - --- - - name: test - hosts: localhost - vars: - context: - app: my_app - environment: production - tasks: - - - name: "Test credstash lookup plugin -- get the password with a context passed as a variable" - debug: msg="{{ lookup('credstash', 'some-password', context=context) }}" - - - name: "Test credstash lookup plugin -- get the password with a context defined here" - debug: msg="{{ lookup('credstash', 'some-password', context=dict(app='my_app', environment='production')) }}" - -If you're not using 2.0 yet, you can do something similar with the credstash tool and the pipe lookup (see below):: - - debug: msg="Poor man's credstash lookup! {{ lookup('pipe', 'credstash -r us-west-1 get my-other-password') }}" - -.. _dns_lookup: - -The DNS Lookup (dig) -```````````````````` -.. versionadded:: 1.9.0 - -.. warning:: This lookup depends on the `dnspython `_ - library. - -The ``dig`` lookup runs queries against DNS servers to retrieve DNS records for -a specific name (*FQDN* - fully qualified domain name). It is possible to lookup any DNS record in this manner. - -There is a couple of different syntaxes that can be used to specify what record -should be retrieved, and for which name. It is also possible to explicitly -specify the DNS server(s) to use for lookups. - -In its simplest form, the ``dig`` lookup plugin can be used to retrieve an IPv4 -address (DNS ``A`` record) associated with *FQDN*: - -.. note:: If you need to obtain the ``AAAA`` record (IPv6 address), you must - specify the record type explicitly. Syntax for specifying the record - type is described below. - -.. note:: The trailing dot in most of the examples listed is purely optional, - but is specified for completeness/correctness sake. - -:: - - - debug: msg="The IPv4 address for example.com. is {{ lookup('dig', 'example.com.')}}" - -In addition to (default) ``A`` record, it is also possible to specify a different -record type that should be queried. This can be done by either passing-in -additional parameter of format ``qtype=TYPE`` to the ``dig`` lookup, or by -appending ``/TYPE`` to the *FQDN* being queried. For example:: - - - debug: msg="The TXT record for example.org. is {{ lookup('dig', 'example.org.', 'qtype=TXT') }}" - - debug: msg="The TXT record for example.org. is {{ lookup('dig', 'example.org./TXT') }}" - -If multiple values are associated with the requested record, the results will be -returned as a comma-separated list. In such cases you may want to pass option -``wantlist=True`` to the plugin, which will result in the record values being -returned as a list over which you can iterate later on:: - - - debug: msg="One of the MX records for gmail.com. is {{ item }}" - with_items: "{{ lookup('dig', 'gmail.com./MX', wantlist=True) }}" - -In case of reverse DNS lookups (``PTR`` records), you can also use a convenience -syntax of format ``IP_ADDRESS/PTR``. The following three lines would produce the -same output:: - - - debug: msg="Reverse DNS for 192.0.2.5 is {{ lookup('dig', '192.0.2.5/PTR') }}" - - debug: msg="Reverse DNS for 192.0.2.5 is {{ lookup('dig', '5.2.0.192.in-addr.arpa./PTR') }}" - - debug: msg="Reverse DNS for 192.0.2.5 is {{ lookup('dig', '5.2.0.192.in-addr.arpa.', 'qtype=PTR') }}" - -By default, the lookup will rely on system-wide configured DNS servers for -performing the query. It is also possible to explicitly specify DNS servers to -query using the ``@DNS_SERVER_1,DNS_SERVER_2,...,DNS_SERVER_N`` notation. This -needs to be passed-in as an additional parameter to the lookup. For example:: - - - debug: msg="Querying 198.51.100.23 for IPv4 address for example.com. produces {{ lookup('dig', 'example.com', '@198.51.100.23') }}" - -In some cases the DNS records may hold a more complex data structure, or it may -be useful to obtain the results in a form of a dictionary for future -processing. The ``dig`` lookup supports parsing of a number of such records, -with the result being returned as a dictionary. This way it is possible to -easily access such nested data. This return format can be requested by -passing-in the ``flat=0`` option to the lookup. For example:: - - - debug: msg="XMPP service for gmail.com. is available at {{ item.target }} on port {{ item.port }}" - with_items: "{{ lookup('dig', '_xmpp-server._tcp.gmail.com./SRV', 'flat=0', wantlist=True) }}" - -Take note that due to the way Ansible lookups work, you must pass the -``wantlist=True`` argument to the lookup, otherwise Ansible will report errors. - -Currently the dictionary results are supported for the following records: - -.. note:: *ALL* is not a record per-se, merely the listed fields are available - for any record results you retrieve in the form of a dictionary. - -========== ============================================================================= -Record Fields ----------- ----------------------------------------------------------------------------- -*ALL* owner, ttl, type -A address -AAAA address -CNAME target -DNAME target -DLV algorithm, digest_type, key_tag, digest -DNSKEY flags, algorithm, protocol, key -DS algorithm, digest_type, key_tag, digest -HINFO cpu, os -LOC latitude, longitude, altitude, size, horizontal_precision, vertical_precision -MX preference, exchange -NAPTR order, preference, flags, service, regexp, replacement -NS target -NSEC3PARAM algorithm, flags, iterations, salt -PTR target -RP mbox, txt -SOA mname, rname, serial, refresh, retry, expire, minimum -SPF strings -SRV priority, weight, port, target -SSHFP algorithm, fp_type, fingerprint -TLSA usage, selector, mtype, cert -TXT strings -========== ============================================================================= - -.. _mongodb_lookup: - -MongoDB Lookup -`````````````` -.. versionadded:: 2.3 - -.. warning:: This lookup depends on the `pymongo 2.4+ `_ - library. - - -The ``MongoDB`` lookup runs the *find()* command on a given *collection* on a given *MongoDB* server. - -The result is a list of jsons, so slightly different from what PyMongo returns. In particular, *timestamps* are converted to epoch integers. - -Currently, the following parameters are supported. - -=========================== ========= ======= ==================== ======================================================================================================================================================================= -Parameter Mandatory Type Default Value Comment ---------------------------- --------- ------- -------------------- ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- -connection_string no string mongodb://localhost/ Can be any valid MongoDB connection string, supporting authentication, replicasets, etc. More info at https://docs.mongodb.org/manual/reference/connection-string/ -extra_connection_parameters no dict {} Dictionary with extra parameters like ssl, ssl_keyfile, maxPoolSize etc... Check the full list here: https://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient -database yes string Name of the database which the query will be made -collection yes string Name of the collection which the query will be made -filter no dict [pymongo default] Criteria of the output Example: { "hostname": "batman" } -projection no dict [pymongo default] Fields you want returned. Example: { "pid": True , "_id" : False , "hostname" : True } -skip no integer [pymongo default] How many results should be skept -limit no integer [pymongo default] How many results should be shown -sort no list [pymongo default] Sorting rules. Please notice the constats are replaced by strings. [ [ "startTime" , "ASCENDING" ] , [ "age", "DESCENDING" ] ] -[any find() parameter] no [any] [pymongo default] Every parameter with exception to *connection_string*, *database* and *collection* are passed to pymongo directly. -=========================== ========= ======= ==================== ======================================================================================================================================================================= - -Please check https://api.mongodb.org/python/current/api/pymongo/collection.html?highlight=find#pymongo.collection.Collection.find for more detais. - -Since there are too many parameters for this lookup method, below is a sample playbook which shows its usage and a nice way to feed the parameters: - -.. code-block:: yaml - - --- - - hosts: all - gather_facts: false - - vars: - mongodb_parameters: - #optional parameter, default = "mongodb://localhost/" - # connection_string: "mongodb://localhost/" - # extra_connection_parameters: { "ssl" : True , "ssl_certfile": /etc/self_signed_certificate.pem" } - - #mandatory parameters - database: 'local' - collection: "startup_log" - - #optional query parameters - #we accept any parameter from the normal mongodb query. - # the official documentation is here - # https://api.mongodb.org/python/current/api/pymongo/collection.html?highlight=find#pymongo.collection.Collection.find - # filter: { "hostname": "batman" } - projection: { "pid": True , "_id" : False , "hostname" : True } - # skip: 0 - limit: 1 - # sort: [ [ "startTime" , "ASCENDING" ] , [ "age", "DESCENDING" ] ] - - tasks: - - debug: msg="Mongo has already started with the following PID [{{ item.pid }}]" - with_mongodb: "{{mongodb_parameters}}" - - - -Sample output: - -.. code-block:: shell-session - - mdiez@batman:~/ansible$ ansible-playbook m.yml -i localhost.ini - - PLAY [all] ********************************************************************* - - TASK [debug] ******************************************************************* - Sunday 20 March 2016 22:40:39 +0200 (0:00:00.023) 0:00:00.023 ********** - ok: [localhost] => (item={u'hostname': u'batman', u'pid': 60639L}) => { - "item": { - "hostname": "batman", - "pid": 60639 - }, - "msg": "Mongo has already started with the following PID [60639]" - } - - PLAY RECAP ********************************************************************* - localhost : ok=1 changed=0 unreachable=0 failed=0 - - Sunday 20 March 2016 22:40:39 +0200 (0:00:00.067) 0:00:00.091 ********** - =============================================================================== - debug ------------------------------------------------------------------- 0.07s - mdiez@batman:~/ansible$ - - -.. _more_lookups: - -More Lookups -```````````` - -Various *lookup plugins* allow additional ways to iterate over data. In :doc:`Loops ` you will learn -how to use them to walk over collections of numerous types. However, they can also be used to pull in data -from remote sources, such as shell commands or even key value stores. This section will cover lookup plugins in this capacity. - -Here are some examples:: - - --- - - hosts: all - - tasks: - - - debug: msg="{{ lookup('env','HOME') }} is an environment variable" - - - name: lines will iterate over each line from stdout of a command - debug: msg="{{ item }} is a line from the result of this command" - with_lines: cat /etc/motd - - - debug: msg="{{ lookup('pipe','date') }} is the raw result of running this command" - - - name: Always use quote filter to make sure your variables are safe to use with shell - debug: msg="{{ lookup('pipe','getent ' + myuser|quote ) }}" - - - name: Quote variables with_lines also as it executes shell - debug: msg="{{ item }} is a line from myfile" - with_lines: "cat {{myfile|quote}}" - - - name: redis_kv lookup requires the Python redis package - debug: msg="{{ lookup('redis_kv', 'redis://localhost:6379,somekey') }} is value in Redis for somekey" - - - name: dnstxt lookup requires the Python dnspython package - debug: msg="{{ lookup('dnstxt', 'example.com') }} is a DNS TXT record for example.com" - - - debug: msg="{{ lookup('template', './some_template.j2') }} is a value from evaluation of this template" - - # Since 2.4, you can pass in variables during evaluation - - debug: msg="{{ lookup('template', './some_template.j2', template_vars=dict(x=42)) }} is evaluated with x=42" - - - name: loading a json file from a template as a string - debug: msg="{{ lookup('template', './some_json.json.j2', convert_data=False) }} is a value from evaluation of this template" - - - - debug: msg="{{ lookup('etcd', 'foo') }} is a value from a locally running etcd" - - # shelvefile lookup retrieves a string value corresponding to a key inside a Python shelve file - - debug: msg="{{ lookup('shelvefile', 'file=path_to_some_shelve_file.db key=key_to_retrieve') }} - - # The following lookups were added in 1.9 - # url lookup splits lines by default, an option to disable this was added in 2.4 - - debug: msg="{{item}}" - with_url: - - 'https://github.com/gremlin.keys' - - # outputs the cartesian product of the supplied lists - - debug: msg="{{item}}" - with_cartesian: - - "{{list1}}" - - "{{list2}}" - - [1,2,3,4,5,6] - - - name: Added in 2.3 allows using the system's keyring - debug: msg={{lookup('keyring','myservice myuser')}} - - -As an alternative, you can also assign lookup plugins to variables or use them elsewhere. -These macros are evaluated each time they are used in a task (or template):: +One way of using lookups is to populate variables. These macros are evaluated each time they are used in a task (or template):: vars: motd_value: "{{ lookup('file', '/etc/motd') }}" - tasks: - - debug: msg="motd value is {{ motd_value }}" .. seealso:: diff --git a/docs/docsite/rst/playbooks_loops.rst b/docs/docsite/rst/playbooks_loops.rst index 5990cbf460e..f8e283ddcd5 100644 --- a/docs/docsite/rst/playbooks_loops.rst +++ b/docs/docsite/rst/playbooks_loops.rst @@ -20,13 +20,13 @@ To save some typing, repeated tasks can be written in short-hand like so:: name: "{{ item }}" state: present groups: "wheel" - with_items: + loop: - testuser1 - testuser2 If you have defined a YAML list in a variables file, or the 'vars' section, you can also do:: - with_items: "{{ somelist }}" + loop: "{{ somelist }}" The above would be the equivalent of:: @@ -41,9 +41,20 @@ The above would be the equivalent of:: state: present groups: "wheel" -The yum and apt modules use with_items to execute fewer package manager transactions. +.. note:: Before 2.5 Ansible mainly used the `with_` keywords to create loops, the `loop` keyword is basically analogous to `with_list`. -Note that the types of items you iterate over with 'with_items' do not have to be simple lists of strings. + +Some plugins like, the yum and apt modules can take lists directly to their options, this is more optimal than looping over the task. +See each action's documentation for details, for now here is an example:: + + - name: optimal yum + yum: name={{list_of_packages}} state=present + + - name: non optimal yum, not only slower but might cause issues with interdependencies + yum: name={{item}} state=present + loop: "{{list_of_packages}}" + +Note that the types of items you iterate over do not have to be simple lists of strings. If you have a list of hashes, you can reference subkeys using things like:: - name: add several users @@ -51,45 +62,21 @@ If you have a list of hashes, you can reference subkeys using things like:: name: "{{ item.name }}" state: present groups: "{{ item.groups }}" - with_items: + loop: - { name: 'testuser1', groups: 'wheel' } - { name: 'testuser2', groups: 'root' } -Also be aware that when combining `when` with `with_items` (or any other loop statement), the `when` statement is processed separately for each item. See :ref:`the_when_statement` for an example. +Also be aware that when combining :ref:`when: playbooks_conditionals` with a loop, the ``when:`` statement is processed separately for each item. +See :ref:`the_when_statement` for an example. -Loops are actually a combination of things `with_` + `lookup()`, so any lookup plugin can be used as a source for a loop, 'items' is lookup. -Please note that ``with_items`` flattens the first depth of the list it is -provided and can yield unexpected results if you pass a list which is composed -of lists. You can work around this by wrapping your nested list inside a list:: +.. _complex_loops: - # This will run debug three times since the list is flattened - - debug: - msg: "{{ item }}" - vars: - nested_list: - - - one - - two - - three - with_items: "{{ nested_list }}" +Complex loops +````````````` - # This will run debug once with the three items - - debug: - msg: "{{ item }}" - vars: - nested_list: - - - one - - two - - three - with_items: - - "{{ nested_list }}" - -.. _nested_loops: - -Nested Loops -```````````` - -Loops can be nested as well:: +Sometimes you need more than what a simple list provides, you can use Jinja2 expressions to create complex lists: +For example, using the 'nested' lookup, you can combine lists:: - name: give users access to multiple databases mysql_user: @@ -97,336 +84,13 @@ Loops can be nested as well:: priv: "{{ item[1] }}.*:ALL" append_privs: yes password: "foo" - with_nested: - - [ 'alice', 'bob' ] - - [ 'clientdb', 'employeedb', 'providerdb' ] + loop: "{{ lookup('nested', [ 'alice', 'bob' ], [ 'clientdb', 'employeedb', 'providerdb' ]) }}" -As with the case of 'with_items' above, you can use previously defined variables.:: - - name: here, 'users' contains the above list of employees - mysql_user: - name: "{{ item[0] }}" - priv: "{{ item[1] }}.*:ALL" - append_privs: yes - password: "foo" - with_nested: - - "{{ users }}" - - [ 'clientdb', 'employeedb', 'providerdb' ] +:doc:`Jinja2 lookups playbooks_lookups`, :doc:`filters playbooks_filters` and :doc:`tests playbooks_tests` +make for some powerful data generation and manipulation. -.. _looping_over_hashes: - -Looping over Hashes -``````````````````` - -.. versionadded:: 1.5 - -Suppose you have the following variable:: - - --- - users: - alice: - name: Alice Appleworth - telephone: 123-456-7890 - bob: - name: Bob Bananarama - telephone: 987-654-3210 - -And you want to print every user's name and phone number. You can loop through the elements of a hash using ``with_dict`` like this:: - - tasks: - - name: Print phone records - debug: - msg: "User {{ item.key }} is {{ item.value.name }} ({{ item.value.telephone }})" - with_dict: "{{ users }}" - -.. _looping_over_fileglobs: - -Looping over Files -`````````````````` - -``with_file`` iterates over the content of a list of files, `item` will be set to the content of each file in sequence. It can be used like this:: - - --- - - hosts: all - - tasks: - - # emit a debug message containing the content of each file. - - debug: - msg: "{{ item }}" - with_file: - - first_example_file - - second_example_file - -Assuming that ``first_example_file`` contained the text "hello" and ``second_example_file`` contained the text "world", this would result in: - -.. code-block:: shell-session - - TASK [debug msg={{ item }}] ****************************************************** - ok: [localhost] => (item=hello) => { - "item": "hello", - "msg": "hello" - } - ok: [localhost] => (item=world) => { - "item": "world", - "msg": "world" - } - -Looping over Fileglobs -`````````````````````` - -``with_fileglob`` matches all files in a single directory, non-recursively, that match a pattern. It calls -`Python's glob library `_, and can be used like this:: - - --- - - hosts: all - - tasks: - - # first ensure our target directory exists - - name: Ensure target directory exists - file: - dest: "/etc/fooapp" - state: directory - - # copy each file over that matches the given pattern - - name: Copy each file over that matches the given pattern - copy: - src: "{{ item }}" - dest: "/etc/fooapp/" - owner: "root" - mode: 0600 - with_fileglob: - - "/playbooks/files/fooapp/*" - -.. note:: When using a relative path with ``with_fileglob`` in a role, Ansible resolves the path relative to the `roles//files` directory. - - -Looping over Filetrees -`````````````````````` - -``with_filetree`` recursively matches all files in a directory tree, enabling you to template a complete tree of files on a target system while retaining permissions and ownership. - -The ``filetree`` lookup-plugin supports directories, files and symlinks, including SELinux and other file properties. Here is a complete list of what each file object consists of: - -* src -* root -* path -* mode -* state -* owner -* group -* seuser -* serole -* setype -* selevel -* uid -* gid -* size -* mtime -* ctime - -If you provide more than one path, it will implement a ``with_first_found`` logic, and will not process entries it already processed in previous paths. This enables the user to merge different trees in order of importance, or add role_vars specific paths to influence different instances of the same role. - -Here is an example of how we use with_filetree within a role. The ``web/`` path is relative to either ``roles//files/`` or ``files/``:: - - --- - - name: Create directories - file: - path: /web/{{ item.path }} - state: directory - mode: '{{ item.mode }}' - with_filetree: web/ - when: item.state == 'directory' - - - name: Template files - template: - src: '{{ item.src }}' - dest: /web/{{ item.path }} - mode: '{{ item.mode }}' - with_filetree: web/ - when: item.state == 'file' - - - name: Recreate symlinks - file: - src: '{{ item.src }}' - dest: /web/{{ item.path }} - state: link - force: yes - mode: '{{ item.mode }}' - with_filetree: web/ - when: item.state == 'link' - - -The following properties are also available: - -* ``root``: allows filtering by original location -* ``path``: contains the relative path to root -* ``uidi``, ``gid``: force-create by exact id, rather than by name -* ``size``, ``mtime``, ``ctime``: filter out files by size, mtime or ctime - - -Looping over Parallel Sets of Data -`````````````````````````````````` - -Suppose you have the following variable data:: - - --- - alpha: [ 'a', 'b', 'c', 'd' ] - numbers: [ 1, 2, 3, 4 ] - -...and you want the set of '(a, 1)' and '(b, 2)'. Use 'with_together' to get this:: - - tasks: - - debug: - msg: "{{ item.0 }} and {{ item.1 }}" - with_together: - - "{{ alpha }}" - - "{{ numbers }}" - -Looping over Subelements -```````````````````````` - -Suppose you want to do something like loop over a list of users, creating them, and allowing them to login by a certain set of -SSH keys. - -In this example, we'll assume you have the following defined and loaded in via "vars_files" or maybe a "group_vars/all" file:: - - --- - users: - - name: alice - authorized: - - /tmp/alice/onekey.pub - - /tmp/alice/twokey.pub - mysql: - password: mysql-password - hosts: - - "%" - - "127.0.0.1" - - "::1" - - "localhost" - privs: - - "*.*:SELECT" - - "DB1.*:ALL" - - name: bob - authorized: - - /tmp/bob/id_rsa.pub - mysql: - password: other-mysql-password - hosts: - - "db1" - privs: - - "*.*:SELECT" - - "DB2.*:ALL" - -You could loop over these subelements like this:: - - - name: Create User - user: - name: "{{ item.name }}" - state: present - generate_ssh_key: yes - with_items: - - "{{ users }}" - - - name: Set authorized ssh key - authorized_key: - user: "{{ item.0.name }}" - key: "{{ lookup('file', item.1) }}" - with_subelements: - - "{{ users }}" - - authorized - -Given the mysql hosts and privs subkey lists, you can also iterate over a list in a nested subkey:: - - - name: Setup MySQL users - mysql_user: - name: "{{ item.0.name }}" - password: "{{ item.0.mysql.password }}" - host: "{{ item.1 }}" - priv: "{{ item.0.mysql.privs | join('/') }}" - with_subelements: - - "{{ users }}" - - mysql.hosts - -Subelements walks a list of hashes (aka dictionaries) and then traverses a list with a given (nested sub-)key inside of those -records. - -Optionally, you can add a third element to the subelements list, that holds a -dictionary of flags. Currently you can add the 'skip_missing' flag. If set to -True, the lookup plugin will skip the lists items that do not contain the given -subkey. Without this flag, or if that flag is set to False, the plugin will -yield an error and complain about the missing subkey. - -The authorized_key pattern is exactly where it comes up most. - -.. _looping_over_integer_sequences: - -Looping over Integer Sequences -`````````````````````````````` - -``with_sequence`` generates a sequence of items. You -can specify a start value, an end value, an optional "stride" value that specifies the number of steps to increment the sequence, and an optional printf-style format string. - -Arguments should be specified as key=value pair strings. - -A simple shortcut form of the arguments string is also accepted: ``[start-]end[/stride][:format]``. - -Numerical values can be specified in decimal, hexadecimal (0x3f8) or octal (0600). -Negative numbers are not supported. This works as follows:: - - --- - - hosts: all - - tasks: - - # create groups - - group: - name: "evens" - state: present - - group: - name: "odds" - state: present - - # create some test users - - user: - name: "{{ item }}" - state: present - groups: "evens" - with_sequence: start=0 end=32 format=testuser%02x - - # create a series of directories with even numbers for some reason - - file: - dest: "/var/stuff/{{ item }}" - state: directory - with_sequence: start=4 end=16 stride=2 - - # a simpler way to use the sequence plugin - # create 4 groups - - group: - name: "group{{ item }}" - state: present - with_sequence: count=4 - -.. _playbooks_loops_random_choice: - -Random Choices -`````````````` - -The 'random_choice' feature can be used to pick something at random. While it's not a load balancer (there are modules -for those), it can somewhat be used as a poor man's load balancer in a MacGyver like situation:: - - - debug: - msg: "{{ item }}" - with_random_choice: - - "go through the door" - - "drink from the goblet" - - "press the red button" - - "do nothing" - -One of the provided strings will be selected at random. - -At a more basic level, they can be used to add chaos and excitement to otherwise predictable automation environments. +.. note:: `with_` loops are actually a combination of things `with_` + `lookup()`, even 'items' is a lookup. `loop` can be used in the same way as shown above. .. _do_until_loops: @@ -451,179 +115,6 @@ The registered variable will also have a new key "attempts" which will have the .. note:: If the "until" parameter isn't defined, the value for the "retries" parameter is forced to 1. -.. _with_first_found: - -Finding First Matched Files -``````````````````````````` - -.. note:: This is an uncommon thing to want to do, but we're documenting it for completeness. You probably won't be reaching for this one often. - -This isn't exactly a loop, but it's close. What if you want to use a reference to a file based on the first file found -that matches a given criteria, and some of the filenames are determined by variable names? Yes, you can do that as follows:: - - - name: INTERFACES | Create Ansible header for /etc/network/interfaces - template: - src: "{{ item }}" - dest: "/etc/foo.conf" - with_first_found: - - "{{ ansible_virtualization_type }}_foo.conf" - - "default_foo.conf" - -This tool also has a long form version that allows for configurable search paths. Here's an example:: - - - name: some configuration template - template: - src: "{{ item }}" - dest: "/etc/file.cfg" - mode: 0444 - owner: "root" - group: "root" - with_first_found: - - files: - - "{{ inventory_hostname }}/etc/file.cfg" - paths: - - ../../../templates.overwrites - - ../../../templates - - files: - - etc/file.cfg - paths: - - templates - -.. _looping_over_the_results_of_a_program_execution: - -Iterating Over The Results of a Program Execution -````````````````````````````````````````````````` - -.. note:: This is an uncommon thing to want to do, but we're documenting it for completeness. You probably won't be reaching for this one often. - -Sometimes you might want to execute a program, and based on the output of that program, loop over the results of that line by line. -Ansible provides a neat way to do that, though you should remember, this is always executed on the control machine, not the remote -machine:: - - - name: Example of looping over a command result - shell: "/usr/bin/frobnicate {{ item }}" - with_lines: - - "/usr/bin/frobnications_per_host --param {{ inventory_hostname }}" - -Ok, that was a bit arbitrary. In fact, if you're doing something that is inventory related you might just want to write a dynamic -inventory source instead (see :doc:`intro_dynamic_inventory`), but this can be occasionally useful in quick-and-dirty implementations. - -Should you ever need to execute a command remotely, you would not use the above method. Instead do this:: - - - name: Example of looping over a REMOTE command result - shell: "/usr/bin/something" - register: command_result - - - name: Do something with each result - shell: "/usr/bin/something_else --param {{ item }}" - with_items: - - "{{ command_result.stdout_lines }}" - -.. _indexed_lists: - -Looping Over A List With An Index -````````````````````````````````` - -.. note:: This is an uncommon thing to want to do, but we're documenting it for completeness. You probably won't be reaching for this one often. - -.. versionadded:: 1.3 - -If you want to loop over an array and also get the numeric index of where you are in the array as you go, you can also do that. -It's uncommonly used:: - - - name: indexed loop demo - debug: - msg: "at array position {{ item.0 }} there is a value {{ item.1 }}" - with_indexed_items: - - "{{ some_list }}" - -.. _using_ini_with_a_loop: - -Using ini file with a loop -`````````````````````````` -.. versionadded:: 2.0 - -The ini plugin can use regexp to retrieve a set of keys. As a consequence, we can loop over this set. Here is the ini file we'll use: - -.. code-block:: ini - - [section1] - value1=section1/value1 - value2=section1/value2 - - [section2] - value1=section2/value1 - value2=section2/value2 - -Here is an example of using ``with_ini``:: - - - debug: - msg: "{{ item }}" - with_ini: - - value[1-2] - - section: section1 - - file: "lookup.ini" - - re: true - -And here is the returned value:: - - { - "changed": false, - "msg": "All items completed", - "results": [ - { - "invocation": { - "module_args": "msg=\"section1/value1\"", - "module_name": "debug" - }, - "item": "section1/value1", - "msg": "section1/value1", - "verbose_always": true - }, - { - "invocation": { - "module_args": "msg=\"section1/value2\"", - "module_name": "debug" - }, - "item": "section1/value2", - "msg": "section1/value2", - "verbose_always": true - } - ] - } - -.. _flattening_a_list: - -Flattening A List -````````````````` - -.. note:: This is an uncommon thing to want to do, but we're documenting it for completeness. You probably won't be reaching for this one often. - -In rare instances you might have several lists of lists, and you just want to iterate over every item in all of those lists. Assume -a really crazy hypothetical datastructure:: - - ---- - # file: roles/foo/vars/main.yml - packages_base: - - [ 'foo-package', 'bar-package' ] - packages_apps: - - [ ['one-package', 'two-package' ]] - - [ ['red-package'], ['blue-package']] - -As you can see the formatting of packages in these lists is all over the place. How can we install all of the packages in both lists?:: - - - name: flattened loop demo - yum: - name: "{{ item }}" - state: present - with_flattened: - - "{{ packages_base }}" - - "{{ packages_apps }}" - -That's how! - -.. _using_register_with_a_loop: - Using register with a loop `````````````````````````` @@ -632,7 +123,7 @@ After using ``register`` with a loop, the data structure placed in the variable Here is an example of using ``register`` with ``with_items``:: - shell: "echo {{ item }}" - with_items: + loop: - "one" - "two" register: echo @@ -682,12 +173,12 @@ Subsequent loops over the registered variable to inspect the results may look li fail: msg: "The command ({{ item.cmd }}) did not have a 0 return code" when: item.rc != 0 - with_items: "{{ echo.results }}" + loop: "{{ echo.results }}" During iteration, the result of the current item will be placed in the variable:: - shell: echo "{{ item }}" - with_items: + loop: - one - two register: echo @@ -695,24 +186,22 @@ During iteration, the result of the current item will be placed in the variable: -.. _looping_over_the_inventory: - Looping over the inventory `````````````````````````` If you wish to loop over the inventory, or just a subset of it, there is multiple ways. -One can use a regular ``with_items`` with the ``ansible_play_batch`` or ``groups`` variables, like this:: +One can use a regular ``loop`` with the ``ansible_play_batch`` or ``groups`` variables, like this:: # show all the hosts in the inventory - debug: msg: "{{ item }}" - with_items: + loop: - "{{ groups['all'] }}" # show all the hosts in the current play - debug: msg: "{{ item }}" - with_items: + loop: - "{{ ansible_play_batch }}" There is also a specific lookup plugin ``inventory_hostnames`` that can be used like this:: @@ -720,14 +209,12 @@ There is also a specific lookup plugin ``inventory_hostnames`` that can be used # show all the hosts in the inventory - debug: msg: "{{ item }}" - with_inventory_hostnames: - - all + loop: "{{lookup('inventory_hostnames', 'all'}}" # show all the hosts matching the pattern, ie all but the group www - debug: msg: "{{ item }}" - with_inventory_hostnames: - - all:!www + loop: "{{lookup('inventory_hostnames', 'all!www'}}" More information on the patterns can be found on :doc:`intro_patterns` @@ -738,13 +225,14 @@ Loop Control .. versionadded:: 2.1 -In 2.0 you are again able to use `with_` loops and task includes (but not playbook includes). This adds the ability to loop over the set of tasks in one shot. +In 2.0 you are again able to use loops and task includes (but not playbook includes). This adds the ability to loop over the set of tasks in one shot. Ansible by default sets the loop variable `item` for each loop, which causes these nested loops to overwrite the value of `item` from the "outer" loops. As of Ansible 2.1, the `loop_control` option can be used to specify the name of the variable to be used for the loop:: # main.yml + - include: inner.yml - include_tasks: inner.yml - with_items: + loop: - 1 - 2 - 3 @@ -754,7 +242,7 @@ As of Ansible 2.1, the `loop_control` option can be used to specify the name of # inner.yml - debug: msg: "outer item={{ outer_item }} inner item={{ item }}" - with_items: + loop: - a - b - c @@ -769,7 +257,7 @@ When using complex data structures for looping the display might get a bit too " digital_ocean: name: "{{ item.name }}" state: present - with_items: + loop: - name: server1 disks: 3gb ram: 15Gb @@ -791,7 +279,7 @@ Another option to loop control is C(pause), which allows you to control the time digital_ocean: name: "{{ item }}" state: present - with_items: + loop: - server1 - server2 loop_control: @@ -808,7 +296,7 @@ for `item`:: # main.yml - include_tasks: inner.yml - with_items: + loop: - 1 - 2 - 3 @@ -819,19 +307,12 @@ for `item`:: - debug: msg: "outer item={{ outer_item }} inner item={{ item }}" - with_items: + loop: - a - b - c - -.. _writing_your_own_iterators: - -Writing Your Own Iterators -`````````````````````````` - -While you ordinarily shouldn't have to, should you wish to write your own ways to loop over arbitrary data structures, you can read :doc:`dev_guide/developing_plugins` for some starter -information. Each of the above features are implemented as plugins in ansible, so there are many implementations to reference. +.. note:: `include` is deprecated, you should be using `include_tasks`, `import_tasks`, `import_play` instead. .. seealso:: diff --git a/docs/docsite/rst/playbooks_python_version.rst b/docs/docsite/rst/playbooks_python_version.rst index 3d4a310889d..40544dc54cc 100644 --- a/docs/docsite/rst/playbooks_python_version.rst +++ b/docs/docsite/rst/playbooks_python_version.rst @@ -36,9 +36,9 @@ using the :func:`list ` filter whenever using :meth:`dict.keys`, - debug: msg: '{{ item }}' # Only works with Python 2 - #with_items: "{{ hosts.keys() }}" + #loop: "{{ hosts.keys() }}" # Works with both Python 2 and Python 3 - with_items: "{{ hosts.keys() | list }}" + loop: "{{ hosts.keys() | list }}" .. _pb-py-compat-iteritems: @@ -59,9 +59,9 @@ compatible with both Python2 and Python3:: - debug: msg: '{{ item }}' # Only works with Python 2 - #with_items: "{{ hosts.iteritems() }}" + #loop: "{{ hosts.iteritems() }}" # Works with both Python 2 and Python 3 - with_items: "{{ hosts.items() | list }}" + loop: "{{ hosts.items() | list }}" .. seealso:: * The :ref:`pb-py-compat-dict-views` entry for information on diff --git a/docs/docsite/rst/playbooks_tags.rst b/docs/docsite/rst/playbooks_tags.rst index 0867eaa5bda..311b36a7f8b 100644 --- a/docs/docsite/rst/playbooks_tags.rst +++ b/docs/docsite/rst/playbooks_tags.rst @@ -12,7 +12,7 @@ Example:: tasks: - yum: name={{ item }} state=installed - with_items: + loop: - httpd - memcached tags: diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index cd84a55a454..08b6d4deb33 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -189,20 +189,20 @@ class TaskExecutor: templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars) items = None - if self._task.loop: - if self._task.loop in self._shared_loader_obj.lookup_loader: + if self._task.loop_with: + if self._task.loop_with in self._shared_loader_obj.lookup_loader: fail = True - if self._task.loop == 'first_found': + if self._task.loop_with == 'first_found': # first_found loops are special. If the item is undefined then we want to fall through to the next value rather than failing. fail = False - loop_terms = listify_lookup_plugin_terms(terms=self._task.loop_args, templar=templar, loader=self._loader, fail_on_undefined=fail, + loop_terms = listify_lookup_plugin_terms(terms=self._task.loop, templar=templar, loader=self._loader, fail_on_undefined=fail, convert_bare=False) if not fail: loop_terms = [t for t in loop_terms if not templar._contains_vars(t)] # get lookup - mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop, loader=self._loader, templar=templar) + mylookup = self._shared_loader_obj.lookup_loader.get(self._task.loop_with, loader=self._loader, templar=templar) # give lookup task 'context' for subdir (mostly needed for first_found) for subdir in ['template', 'var', 'file']: # TODO: move this to constants? @@ -213,7 +213,12 @@ class TaskExecutor: # run lookup items = mylookup.run(terms=loop_terms, variables=self._job_vars, wantlist=True) else: - raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop) + raise AnsibleError("Unexpected failure in finding the lookup named '%s' in the available lookup plugins" % self._task.loop_with) + + elif self._task.loop: + items = templar.template(self._task.loop) + if not isinstance(items, list): + raise AnsibleError("Invalid data passed to 'loop' it requires a list, got this instead: %s" % items) # now we restore any old job variables that may have been modified, # and delete them if they were in the play context vars but not in @@ -264,7 +269,10 @@ class TaskExecutor: u" to something else to avoid variable collisions and unexpected behavior." % loop_var) ran_once = False - items = self._squash_items(items, loop_var, task_vars) + if self._task.loop_with: + # Only squash with 'with_:' not with the 'loop:', 'magic' squashing can be removed once with_ loops are + items = self._squash_items(items, loop_var, task_vars) + for item in items: task_vars[loop_var] = item diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index 9ed4c46c620..e19d8b60e3e 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -75,8 +75,7 @@ class Task(Base, Conditional, Taggable, Become): _delegate_to = FieldAttribute(isa='string') _delegate_facts = FieldAttribute(isa='bool', default=False) _failed_when = FieldAttribute(isa='list', default=[]) - _loop = FieldAttribute(isa='string', private=True, inherit=False) - _loop_args = FieldAttribute(isa='list', private=True, inherit=False) + _loop = FieldAttribute() _loop_control = FieldAttribute(isa='class', class_type=LoopControl, inherit=False) _name = FieldAttribute(isa='string', default='') _notify = FieldAttribute(isa='list') @@ -85,6 +84,9 @@ class Task(Base, Conditional, Taggable, Become): _retries = FieldAttribute(isa='int', default=3) _until = FieldAttribute(isa='list', default=[]) + # deprecated, used to be loop and loop_args but loop has been repurposed + _loop_with = FieldAttribute(isa='string', private=True, inherit=False) + def __init__(self, block=None, role=None, task_include=None): ''' constructors a task, without the Task.load classmethod, it will be pretty blank ''' @@ -145,16 +147,17 @@ class Task(Base, Conditional, Taggable, Become): else: return "TASK: %s" % self.get_name() - def _preprocess_loop(self, ds, new_ds, k, v): + def _preprocess_with_loop(self, ds, new_ds, k, v): ''' take a lookup plugin name and store it correctly ''' loop_name = k.replace("with_", "") - if new_ds.get('loop') is not None: + if new_ds.get('loop') is not None or new_ds.get('loop_with') is not None: raise AnsibleError("duplicate loop in task: %s" % loop_name, obj=ds) if v is None: raise AnsibleError("you must specify a value when using %s" % k, obj=ds) - new_ds['loop'] = loop_name - new_ds['loop_args'] = v + new_ds['loop_with'] = loop_name + new_ds['loop'] = v + display.deprecated("with_ type loops are being phased out, use the 'loop' keyword instead", version="2.9") def preprocess_data(self, ds): ''' @@ -210,7 +213,7 @@ class Task(Base, Conditional, Taggable, Become): continue elif k.replace("with_", "") in lookup_loader: # transform into loop property - self._preprocess_loop(ds, new_ds, k, v) + self._preprocess_with_loop(ds, new_ds, k, v) else: # pre-2.0 syntax allowed variables for include statements at the top level of the task, # so we move those into the 'vars' dictionary here, and show a deprecation message @@ -248,9 +251,9 @@ class Task(Base, Conditional, Taggable, Become): super(Task, self).post_validate(templar) - def _post_validate_loop_args(self, attr, value, templar): + def _post_validate_loop(self, attr, value, templar): ''' - Override post validation for the loop args field, which is templated + Override post validation for the loop field, which is templated specially in the TaskExecutor class when evaluating loops. ''' return value diff --git a/lib/ansible/vars/manager.py b/lib/ansible/vars/manager.py index 6ee568dce30..4f5e6668802 100644 --- a/lib/ansible/vars/manager.py +++ b/lib/ansible/vars/manager.py @@ -522,7 +522,7 @@ class VariableManager: if task.loop is not None: if task.loop in lookup_loader: try: - loop_terms = listify_lookup_plugin_terms(terms=task.loop_args, templar=templar, + loop_terms = listify_lookup_plugin_terms(terms=task.loop, templar=templar, loader=self._loader, fail_on_undefined=True, convert_bare=False) items = lookup_loader.get(task.loop, loader=self._loader, templar=templar).run(terms=loop_terms, variables=vars_copy) except AnsibleUndefinedVariable: diff --git a/test/units/executor/test_task_executor.py b/test/units/executor/test_task_executor.py index 3b75636b822..fc69f2dbc15 100644 --- a/test/units/executor/test_task_executor.py +++ b/test/units/executor/test_task_executor.py @@ -106,8 +106,8 @@ class TestTaskExecutor(unittest.TestCase): mock_host = MagicMock() mock_task = MagicMock() - mock_task.loop = 'items' - mock_task.loop_args = ['a', 'b', 'c'] + mock_task.loop_with = 'items' + mock_task.loop = ['a', 'b', 'c'] mock_play_context = MagicMock()