Lookup docs (#30280)

* finalize lookup documentation
* minor fixes to ansible-doc
 - actually show which file caused error on when listing plugins
 - removed redundant display of type and name
* smart quote fixes from toshio

lib/ansible/cli/doc.py

@@ -28,6 +28,7 @@ import yaml
 from ansible import constants as C
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
+from ansible.module_utils._text import to_native
 from ansible.module_utils.six import string_types
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.plugins.loader import module_loader, action_loader, lookup_loader, callback_loader, cache_loader, \
@@ -227,35 +228,38 @@ class DocCLI(CLI):
         deprecated = []
         for plugin in sorted(self.plugin_list):
 
-            # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
-            filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True, check_aliases=True)
-
-            if filename is None:
-                continue
-            if filename.endswith(".ps1"):
-                continue
-            if os.path.isdir(filename):
-                continue
-
-            doc = None
             try:
-                doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(filename)
+                # if the module lives in a non-python file (eg, win_X.ps1), require the corresponding python file for docs
-            except:
+                filename = loader.find_plugin(plugin, mod_type='.py', ignore_deprecated=True, check_aliases=True)
-                display.warning("%s has a documentation formatting error" % plugin)
 
-            if not doc:
+                if filename is None:
-                desc = 'UNDOCUMENTED'
+                    continue
-                display.warning("%s parsing did not produce documentation." % plugin)
+                if filename.endswith(".ps1"):
-            else:
+                    continue
-                desc = self.tty_ify(doc.get('short_description', '?')).strip()
+                if os.path.isdir(filename):
+                    continue
 
-            if len(desc) > linelimit:
+                doc = None
-                desc = desc[:linelimit] + '...'
+                try:
+                    doc, plainexamples, returndocs, metadata = plugin_docs.get_docstring(filename)
+                except:
+                    display.warning("%s has a documentation formatting error" % plugin)
 
-            if plugin.startswith('_'):  # Handle deprecated
+                if not doc or not isinstance(doc, dict):
-                deprecated.append("%-*s %-*.*s" % (displace, plugin[1:], linelimit, len(desc), desc))
+                    desc = 'UNDOCUMENTED'
-            else:
+                    display.warning("%s parsing did not produce documentation." % plugin)
-                text.append("%-*s %-*.*s" % (displace, plugin, linelimit, len(desc), desc))
+                else:
+                    desc = self.tty_ify(doc.get('short_description', 'INVALID SHORT DESCRIPTION').strip())
+
+                if len(desc) > linelimit:
+                    desc = desc[:linelimit] + '...'
+
+                if plugin.startswith('_'):  # Handle deprecated
+                    deprecated.append("%-*s %-*.*s" % (displace, plugin[1:], linelimit, len(desc), desc))
+                else:
+                    text.append("%-*s %-*.*s" % (displace, plugin, linelimit, len(desc), desc))
+            except Exception as e:
+                raise AnsibleError("Failed reading docs at %s: %s" % (plugin, to_native(e)))
 
         if len(deprecated) > 0:
             text.append("\nDEPRECATED:")
@@ -401,7 +405,7 @@ class DocCLI(CLI):
 
     def get_man_text(self, doc):
 
-        IGNORE = frozenset(['module', 'docuri', 'version_added', 'short_description', 'now_date', 'plainexamples', 'returndocs'])
+        IGNORE = frozenset(['module', 'docuri', 'version_added', 'short_description', 'now_date', 'plainexamples', 'returndocs', self.options.type])
         opt_indent = "        "
         text = []
         pad = display.columns * 0.20

lib/ansible/plugins/lookup/cartesian.py

@@ -1,19 +1,6 @@
 # (c) 2013, Bradley Young <young.bradley@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
@@ -23,22 +10,25 @@ DOCUMENTATION = """
     short_description: returns the cartesian product of lists
     description:
         - Takes the input lists and returns a list that represents the product of the input lists.
+        - It is clearer with an example, it turns [1, 2, 3], [a, b] into [1, a], [1, b], [2, a], [2, b], [3, a], [3, b].
+         You can see the exact syntax in the examples section.
     options:
-        _raw:
+      _raw:
-            description:
+        description:
-                - a set of lists
+          - a set of lists
-            required: True
+        required: True
 """
 
 EXAMPLES = """
+- name: Example of the change in the description
+  debug: msg="{{ [1,2,3]|lookup('cartesian', [a, b])}}"
 
-  - name: outputs the cartesian product of the supplied lists
+- name: loops over the cartesian product of the supplied lists
-    debug: msg="{{item}}"
+  debug: msg="{{item}}"
-    with_cartesian:
+  with_cartesian:
-         - "{{list1}}"
+    - "{{list1}}"
-         - "{{list2}}"
+    - "{{list2}}"
-  - name: used as lookup changes [1, 2, 3], [a, b] into [1, a], [1, b], [2, a], [2, b], [3, a], [3, b]
+    - [1,2,3,4,5,6]
-    debug: msg="{{ [1,2,3]|lookup('cartesian', [a, b])}}"
 """
 
 RETURN = """
@@ -58,7 +48,6 @@ from ansible.utils.listify import listify_lookup_plugin_terms
 class LookupModule(LookupBase):
     """
     Create the cartesian product of lists
-    [1, 2, 3], [a, b] -> [1, a], [1, b], [2, a], [2, b], [3, a], [3, b]
     """
 
     def _lookup_variables(self, terms):

lib/ansible/plugins/lookup/chef_databag.py

@@ -1,39 +1,43 @@
 # (c) 2016, Josh Bradley <jbradley(at)digitalocean.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-"""
-
-USAGE: {{ lookup('chef_databag', 'name=data_bag_name item=data_bag_item') }}
-
-NOTES: This is a lookup plugin to provide access to chef data bags using the
-       pychef package. It interfaces with the chef server api using the same
-       methods to find a knife or chef-client config file to load parameters
-       from, starting from either the given base path or the current working
-       directory. The lookup order mirrors the one from Chef, all folders in
-       the base path are walked back looking for the following configuration
-       file in order : .chef/knife.rb, ~/.chef/knife.rb, /etc/chef/client.rb
-
-Requires: pychef package: https://pychef.readthedocs.io `pip install pychef`
-
-"""
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: chef_databag
+    version_added: "2.3"
+    short_description: fetches data from a Chef Databag
+    description:
+       - "This is a lookup plugin to provide access to chef data bags using the pychef package.
+         It interfaces with the chef server api using the same methods to find a knife or chef-client config file to load parameters from,
+         starting from either the given base path or the current working directory.
+         The lookup order mirrors the one from Chef, all folders in the base path are walked back looking for the following configuration
+         file in order : .chef/knife.rb, ~/.chef/knife.rb, /etc/chef/client.rb"
+    requirements:
+        - "pychef (python library https://pychef.readthedocs.io `pip install pychef`"
+    options:
+        name:
+          description:
+            - Name of the databag
+          required: True
+        item:
+          description:
+            - Item to fetch
+          required: True
+"""
+
+EXAMPLES = """
+    - debug
+        msg: {{ lookup('chef_databag', 'name=data_bag_name item=data_bag_item') }}
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - The value from the databag
+"""
+
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.parsing.splitter import parse_kv

lib/ansible/plugins/lookup/consul_kv.py

@@ -1,64 +1,63 @@
 # (c) 2015, Steve Gargan <steve.gargan@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-'''
+DOCUMENTATION = """
-Lookup plugin to grab metadata from a consul key value store.
+    lookup: cartesian
-============================================================
+    version_added: "1.9"
+    short_description: grab metadata from a consul key value store.
+    description:
+      - lookup metadata for a playbook from the key value store in a consul cluster.
+        Values can be easily set in the kv store with simple rest commands
+      - "curl -X PUT -d 'some-value' http://localhost:8500/v1/kv/ansible/somedata"
+    requirements:
+      - 'python-consul (python library http://python-consul.readthedocs.org/en/latest/#installation)'
+    options:
+      _raw:
+        description: key(s) to retrieve
+        type: list
+        required: True
+      recurse:
+        type: boolean
+        description: if true, will retrieve all the values that have the given key as prefix
+        default: False
+      index:
+        description: if the key has a value with the specified index then this is returned allowing access to historical values.
+      token:
+        description: acl token to allow access to restricted values.
+      host:
+        default: localhost
+        description:
+           - the target to connect to, must be a resolvable address.
+        env:
+          - name: ANSIBLE_CONSUL_URL
+      port:
+        description: the port of the target host to connect to.
+        default: 8500
+"""
 
-Plugin will lookup metadata for a playbook from the key value store in a
+EXAMPLES = """
-consul cluster. Values can be easily set in the kv store with simple rest
+  - debug: msg='key contains {{item}}'
-commands e.g.
+    with_consul_kv:
+      - 'key/to/retrieve'
 
-curl -X PUT -d 'some-value' http://localhost:8500/v1/kv/ansible/somedata
+  - name: Parameters can be provided after the key be more specific about what to retrieve
+    debug: msg='key contains {{item}}'
+    with_consul_kv:
+      - 'key/to recurse=true token=E6C060A9-26FB-407A-B83E-12DDAFCB4D98')}}'
 
-this can then be looked up in a playbook as follows
+  - name: retrieving a KV from a remote cluster on non default port
+    debug:
+      msg: "{{ lookup('consul_kv', 'my/key', host='10.10.10.10', port='2000')
+"""
 
-- debug: msg='key contains {{item}}'
+RETURN = """
-  with_consul_kv:
+  _raw:
-    - 'key/to/retrieve'
+    description:
-
+      - value(s) stored in consul
-
+"""
-Parameters can be provided after the key be more specific about what to retrieve e.g.
-
-- debug: msg='key contains {{item}}'
-  with_consul_kv:
-    - 'key/to recurse=true token=E6C060A9-26FB-407A-B83E-12DDAFCB4D98')}}'
-
-recurse: if true, will retrieve all the values that have the given key as prefix
-index: if the key has a value with the specified index then this is returned
-       allowing access to historical values.
-token: acl token to allow access to restricted values.
-host: the target to connect to, must be a resolvable address. defaults to localhost
-port: the port of the target host to connect to. defaults to 8500
-
-By default this will lookup keys via the consul agent running on http://localhost:8500
-this can be changed by setting the env variable 'ANSIBLE_CONSUL_URL' to point to the url
-of the kv store you'd like to use or by specifying the parameters above.
-
-This is an example of retrieving a KV from a remote cluster on non default port
-
-- debug:
-    msg: "{{ lookup('consul_kv', 'my/key', host='10.10.10.10', port='2000')
-
-'''
-
-######################################################################
 
 import os
 import sys

lib/ansible/plugins/lookup/credstash.py

@@ -1,22 +1,82 @@
 # (c) 2015, Ensighten <infra@ensighten.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: credstash
+    version_added: "2.0"
+    short_description: retrieve secrets from Credstash on AWS
+    requirements:
+      - credstash (python library)
+    description:
+      - Credstash is a small utility for managing secrets using AWS's KMS and DynamoDB: https://github.com/fugue/credstash
+    options:
+      _terms:
+        description: term or list of terms to lookup in the credit store
+        type: list
+        required: True
+      table:
+        description: name of the credstash table to query
+        default: 'credential-store'
+        required: True
+      version:
+        description: Credstash version
+      region:
+        description: AWS region
+      profile_name:
+        description: AWS profile to use for authentication
+        env:
+          - name: AWS_PROFILE
+      aws_access_key_id:
+        description: AWS access key ID
+        env:
+          - name: AWS_ACCESS_KEY_ID
+      aws_secret_access_key:
+        description: AWS access key
+        env:
+          - name: AWS_SECRET_ACCESS_KEY
+      aws_session_token:
+        description: AWS session token
+        env:
+          - name: AWS_SESSION_TOKEN
+"""
+
+EXAMPLES = """
+- name: first use credstash to store your secrets
+  shell: credstash put my-github-password secure123
+
+- name: "Test credstash lookup plugin -- get my github password"
+  debug: msg="Credstash lookup! {{ lookup('credstash', 'my-github-password') }}"
+
+- name: "Test credstash lookup plugin -- get my other password from us-west-1"
+  debug: msg="Credstash lookup! {{ lookup('credstash', 'my-other-password', region='us-west-1') }}"
+
+- name: "Test credstash lookup plugin -- get the company's github password"
+  debug: msg="Credstash lookup! {{ lookup('credstash', 'company-github-password', table='company-passwords') }}"
+
+- name: Example play using the 'context' feature
+  hosts: localhost
+  vars:
+    context:
+      app: my_app
+      environment: production
+  tasks:
+
+  - name: "Test credstash lookup plugin -- get the password with a context passed as a variable"
+    debug: msg="{{ lookup('credstash', 'some-password', context=context) }}"
+
+  - name: "Test credstash lookup plugin -- get the password with a context defined here"
+    debug: msg="{{ lookup('credstash', 'some-password', context=dict(app='my_app', environment='production')) }}"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - value(s) stored in Credstash
+"""
+
 import os
 
 from ansible.errors import AnsibleError
@@ -49,8 +109,7 @@ class LookupModule(LookupBase):
                 aws_session_token = kwargs.pop('aws_session_token', os.getenv('AWS_SESSION_TOKEN', None))
                 kwargs_pass = {'profile_name': profile_name, 'aws_access_key_id': aws_access_key_id,
                                'aws_secret_access_key': aws_secret_access_key, 'aws_session_token': aws_session_token}
-                val = credstash.getSecret(term, version, region, table,
+                val = credstash.getSecret(term, version, region, table, context=kwargs, **kwargs_pass)
-                                          context=kwargs, **kwargs_pass)
             except credstash.ItemNotFound:
                 raise AnsibleError('Key {0} not found'.format(term))
             except Exception as e:

lib/ansible/plugins/lookup/csvfile.py

@@ -1,22 +1,52 @@
 # (c) 2013, Jan-Piet Mens <jpmens(at)gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: csvfile
+    author: Jan-Piet Mens (@jpmens) <jpmens(at)gmail.com>
+    version_added: "1.5"
+    short_description: read data from a TSV or CSV file
+    description:
+      - The csvfile lookup reads the contents of a file in CSV (comma-separated value) format.
+        The lookup looks for the row where the first column matches keyname, and returns the value in the second column, unless a different column is specified.
+    options:
+      col:
+        description:  column to return (0 index).
+        default: "1"
+      default:
+        description: what to return if the value is not found in the file.
+        default: ''
+      delimiter:
+        description: field separator in the file, for a tab you can specify "TAB" or "t".
+        default: TAB
+      file:
+        description: name of the CSV/TSV file to open.
+        default: ansible.csv
+      encoding:
+        description: Encoding (character set) of the used CSV file.
+        default: utf-8
+        version_added: "2.1"
+    notes:
+      - The default is for TSV files (tab delimeted) not CSV (comma delimted) ... yes the name is misleading.
+"""
+
+EXAMPLES = """
+- name:  Match 'Li' on the first column, return the second column (0 based index)
+  debug: msg="The atomic number of Lithium is {{ lookup('csvfile', 'Li file=elements.csv delimiter=,') }}"
+
+- name: msg="Match 'Li' on the first column, but return the 3rd column (columns start counting after the match)"
+  debug: msg="The atomic mass of Lithium is {{ lookup('csvfile', 'Li file=elements.csv delimiter=, col=2') }}"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - value(s) stored in file column
+"""
+
 import codecs
 import csv
 from collections import MutableSequence

lib/ansible/plugins/lookup/cyberarkpassword.py

@@ -1,23 +1,68 @@
 # (c) 2017, Edward Nunez <edward.nunez@cyberark.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: cyberarkpassword
+    version_added: "2.4"
+    short_description: get secrets from CyberArk AIM
+    requirements:
+      - CyberArk AIM tool installed
+    description:
+    options :
+      _command:
+        description: Cyberark CLI utility.
+        env:
+          - name: AIM_CLIPASSWORDSDK_CMD
+        default: '/opt/CARKaim/sdk/clipasswordsdk'
+      appid:
+        description: Defines the unique ID of the application that is issuing the password request.
+        required: True
+      query:
+        description: Describes the filter criteria for the password retrieval.
+        required: True
+      output:
+        description:
+          - Specifies the desired output fields separated by commas.
+          - "They could be: Password, PassProps.<property>, PasswordChangeInProcess"
+        default: 'password'
+      _extra:
+        description: for extra_parms values please check parameters for clipasswordsdk in CyberArk's "Credential Provider and ASCP Implementation Guide"
+    note:
+      - For Ansible on windows, please change the -parameters (-p, -d, and -o) to /parameters (/p, /d, and /o) and change the location of CLIPasswordSDK.exe
+"""
+
+EXAMPLES = """
+  - name: passing options to the lookup
+    debug: msg={{ lookup("cyberarkpassword", cyquery)}}
+    vars:
+      cyquery:
+        appid: "app_ansible"
+        query": "safe=CyberArk_Passwords;folder=root;object=AdminPass"
+        output: "Password,PassProps.UserName,PassProps.Address,PasswordChangeInProcess"
+
+
+  - name: used in a loop
+    debug: msg={{item}}
+    with_cyberarkpassword:
+        appid: 'app_ansible'
+        query: 'safe=CyberArk_Passwords;folder=root;object=AdminPass'
+        output: 'Password,PassProps.UserName,PassProps.Address,PasswordChangeInProcess'
+"""
+
+RETURN = """
+  password:
+    description:
+      - The actual value stored
+  passprops:
+    description: properties assigned to the entry
+    type: dictionary
+  passwordchangeinprocess:
+    description: did the password change?
+"""
+
 import os
 import subprocess
 from subprocess import PIPE
@@ -118,35 +163,6 @@ class LookupModule(LookupBase):
     """
     USAGE:
 
-        {{ lookup("cyberarkpassword", {"appid": "app_ansible", "query": "safe=CyberArk_Passwords;folder=root;object=AdminPass",
-                                       "output": "Password,PassProps.UserName,PassProps.Address,PasswordChangeInProcess"}) }}
-
-        OR
-
-      with_cyberarkpassword:
-        appid: 'app_ansible'
-        query: 'safe=CyberArk_Passwords;folder=root;object=AdminPass'
-        output: 'Password,PassProps.UserName,PassProps.Address,PasswordChangeInProcess'
-
-
-    It Requires CyberArk AIM Installed, and /opt/CARKaim/sdk/clipasswordsdk in place or set environment variable AIM_CLIPASSWORDSDK_CMD to the AIM
-    CLI Password SDK executable.
-
-     Args:
-         appid (str): Defines the unique ID of the application that is issuing the password request.
-         query (str): Describes the filter criteria for the password retrieval.
-         output (str): Specifies the desired output fields separated by commas. They could be: Password, PassProps.<property>, PasswordChangeInProcess
-         Optionally, you can specify extra parameters recognized by clipasswordsdk (like FailRequestOnPasswordChange, Queryformat, Reason, etc.)
-
-     Returns:
-         dict: A dictionary with 'password' as key for the credential, passprops.<property>, passwordchangeinprocess
-               If the specified property does not exist for this password, the value <na> will be returned for this property.
-               If the value of the specified property is empty, <null> will be returned.
-
-
-    for extra_parms values please check parameters for clipasswordsdk in CyberArk's "Credential Provider and ASCP Implementation Guide"
-
-    For Ansible on windows, please change the -parameters (-p, -d, and -o) to /parameters (/p, /d, and /o) and change the location of CLIPasswordSDK.exe
     """
 
     def run(self, terms, variables=None, **kwargs):

lib/ansible/plugins/lookup/dict.py

@@ -1,22 +1,51 @@
 # (c) 2014, Kent R. Spillner <kspillner@acm.org>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: dict
+    version_added: "1.5"
+    short_description: returns key/value pair items from a dictionary
+    description:
+        - Takes a dictionary as input and returns a list with each item in the list being a dictionary with 'key' and 'value' as
+          keys to the previous dictionary's structure.
+    options:
+        _raw:
+            description:
+                - A dictionary
+            required: True
+"""
+
+EXAMPLES = """
+tasks:
+  - name: show dictionary
+    debug: msg="{{item.key}}: {{item.value}}"
+    with_dict: {a: 1, b: 2, c: 3}a
+
+# with predefined vars
+vars:
+  users:
+    alice:
+      name: Alice Appleworth
+      telephone: 123-456-7890
+    bob:
+      name: Bob Bananarama
+      telephone: 987-654-3210
+tasks:
+  - name: Print phone records
+    debug:
+      msg: "User {{ item.key }} is {{ item.value.name }} ({{ item.value.telephone }})"
+    with_dict: "{{ users }}"
+"""
+
+RETURN = """
+  _list:
+    description:
+      - list of composed dictonaries with key and value
+    type: list
+"""
 import collections
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/dig.py

@@ -1,22 +1,102 @@
 # (c) 2015, Jan-Piet Mens <jpmens(at)gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: dig
+    author: Jan-Piet Mens (@jpmens) <jpmens(at)gmail.com>
+    version_added: "1.9"
+    short_description: use 'dig' to query DNS
+    requirements:
+      - dig (CLI utility)
+    description: test
+    lola:
+#      - The dig lookup runs queries against DNS servers to retrieve DNS records for a specific name (FQDN - fully qualified domain name).
+#        It is possible to lookup any DNS record in this manner.
+#      - There is a couple of different syntaxes that can be used to specify what record should be retrieved, and for which name.
+#         It is also possible to explicitly specify the DNS server(s) to use for lookups.
+#      - In its simplest form, the dig lookup plugin can be used to retrieve an IPv4 address (DNS A record) associated with FQDN
+#      - In addition to (default) A record, it is also possible to specify a different record type that should be queried.
+#        This can be done by either passing-in additional parameter of format qtype=TYPE to the dig lookup, or by appending /TYPE to the FQDN being queried.
+#      - If multiple values are associated with the requested record, the results will be returned as a comma-separated list.
+#        In such cases you may want to pass option wantlist=True to the plugin, which will result in the record values being returned as a list
+#        over which you can iterate later on.
+#      - By default, the lookup will rely on system-wide configured DNS servers for performing the query.
+#        It is also possible to explicitly specify DNS servers to query using the @DNS_SERVER_1,DNS_SERVER_2,...,DNS_SERVER_N notation.
+#        This needs to be passed-in as an additional parameter to the lookup
+    options:
+      _terms:
+        description: domain(s) to query
+      qtype:
+        description: record type to query
+        default: 'A'
+        choices: [A, ALL, AAAA, CNAME, DNAME, DLV, DNSKEY, DS, HINFO, LOC, MX, NAPTR, NS, NSEC3PARAM, PTR, RP, RRSIG, SOA, SPF, SRV, SSHFP, TLSA, TXT]
+      flat:
+        description: If 0 each record is returned as a dictionary, otherwise a string
+        default: 1
+    notes:
+      - ALL is not a record per-se, merely the listed fields are available for any record results you retrieve in the form of a dictionary.
+      - While the 'dig' lookup plugin supports anything which dnspython supports out of the box, only a subset can be converted into a dictionary.
+      - If you need to obtain the AAAA record (IPv6 address), you must specify the record type explicitly.
+        Syntax for specifying the record type is shown in the examples below.
+      - The trailing dot in most of the examples listed is purely optional, but is specified for completeness/correctness sake.
+"""
+
+EXAMPLES = """
+- name: Simple A record (IPV4 address) lookup for example.com
+  debug: msg="{{ lookup('dig', 'example.com.')}}"
+
+- name: "The TXT record for example.org."
+  debug: msg="{{ lookup('dig', 'example.org.', 'qtype=TXT') }}"
+
+- name: "The TXT record for example.org, alternative syntax."
+  debug: msg="{{ lookup('dig', 'example.org./TXT') }}"
+
+- name: use in a loop
+  debug: msg="MX record for gmail.com {{ item }}"
+  with_items: "{{ lookup('dig', 'gmail.com./MX', wantlist=True) }}"
+
+- debug: msg="Reverse DNS for 192.0.2.5 is {{ lookup('dig', '192.0.2.5/PTR') }}"
+- debug: msg="Reverse DNS for 192.0.2.5 is {{ lookup('dig', '5.2.0.192.in-addr.arpa./PTR') }}"
+- debug: msg="Reverse DNS for 192.0.2.5 is {{ lookup('dig', '5.2.0.192.in-addr.arpa.', 'qtype=PTR') }}"
+- debug: msg="Querying 198.51.100.23 for IPv4 address for example.com. produces {{ lookup('dig', 'example.com', '@198.51.100.23') }}"
+
+- debug: msg="XMPP service for gmail.com. is available at {{ item.target }} on port {{ item.port }}"
+  with_items: "{{ lookup('dig', '_xmpp-server._tcp.gmail.com./SRV', 'flat=0', wantlist=True) }}"
+"""
+
+RETURN = """
+  _list:
+    description:
+      - list of composed strings or dictonaries with key and value
+        If a dictionary, fields shows the keys returned depending on query type
+    fields:
+       ALL: owner, ttl, type
+       A: address
+       AAAA: address
+       CNAME: target
+       DNAME: target
+       DLV: algorithm, digest_type, key_tag, digest
+       DNSKEY: flags, algorithm, protocol, key
+       DS: algorithm, digest_type, key_tag, digest
+       HINFO:  cpu, os
+       LOC: latitude, longitude, altitude, size, horizontal_precision, vertical_precision
+       MX: preference, exchange
+       NAPTR: order, preference, flags, service, regexp, replacement
+       NS: target
+       NSEC3PARAM: algorithm, flags, iterations, salt
+       PTR: target
+       RP: mbox, txt
+       SOA: mname, rname, serial, refresh, retry, expire, minimum
+       SPF: strings
+       SRV: priority, weight, port, target
+       SSHFP: algorithm, fp_type, fingerprint
+       TLSA: usage, selector, mtype, cert
+       TXT: strings
+"""
+
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils._text import to_native

lib/ansible/plugins/lookup/dnstxt.py

@@ -1,22 +1,48 @@
 # (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: dnstxt
+    author: Jan-Piet Mens (@jpmens) <jpmens(at)gmail.com>
+    version_added: "0.9"
+    short_description: query a domain(s)'s DNS txt fields
+    requirements:
+      - dns/dns.resolver (python library)
+    description:
+      - Uses a python library to return the DNS TXT record for a domain.
+    options:
+      _terms:
+        description: domain or list of domains to query TXT records from
+        required: True
+        type: list
+"""
+
+EXAMPLES = """
+- name: show txt entry
+  debug: msg="{{lookup('dnstxt', ['test.example.com'])}}"
+
+- name: iterate over txt entries
+  debug: msg="{{item}}"
+  with_dnstxt:
+    - 'test.example.com'
+    - 'other.example.com'
+    - 'last.example.com'
+
+- name: iterate of a comma delimited DNS TXT entry
+  debug: msg="{{item}}"
+  with_dnstxt: "{{lookup('dnstxt', ['test.example.com']).split(',')}}"
+"""
+
+RETURN = """
+  _list:
+    description:
+      - values returned by the DNS TXT record.
+    type: list
+"""
+
 HAVE_DNS = False
 try:
     import dns.resolver
@@ -29,7 +55,6 @@ from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_native
 from ansible.plugins.lookup import LookupBase
 
-
 # ==============================================================
 # DNSTXT: DNS TXT records
 #
@@ -37,6 +62,7 @@ from ansible.plugins.lookup import LookupBase
 # TODO: configurable resolver IPs
 # --------------------------------------------------------------
 
+
 class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):

lib/ansible/plugins/lookup/env.py

@@ -1,22 +1,34 @@
 # (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: env
+    author: Jan-Piet Mens (@jpmens) <jpmens(at)gmail.com>
+    version_added: "0.9"
+    short_description: read the value of environment variables
+    requirements:
+      - dns/dns.resolver (python library)
+    description:
+        - Allows you to query the environment variables available on the controller when you invoked Ansible.
+    options:
+      _terms:
+        description: Environment variable or list of them to lookup the values for
+        required: True
+"""
+
+EXAMPLES = """
+- debug: msg="{{ lookup('env','HOME') }} is an environment variable"
+"""
+
+RETURN = """
+  _list:
+    description:
+      - values returned by the DNS TXT record.
+    type: list
+"""
 import os
 
 from ansible.plugins.lookup import LookupBase

lib/ansible/plugins/lookup/etcd.py

@@ -27,7 +27,7 @@ DOCUMENTATION = '''
     description:
         - Retrieves data from an etcd server
     options:
-        _raw:
+        _terms:
             description:
                 - the list of keys to lookup on the etcd server
             type: list

lib/ansible/plugins/lookup/file.py

@@ -1,22 +1,42 @@
 # (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: file
+    author: Daniel Hokka Zakrisson <daniel@hozac.com>
+    version_added: "0.9"
+    short_description: read file contents
+    description:
+        - This lookup returns the contents from a file on the Ansible controller's file system.
+    options:
+      _terms:
+        description: path(s) of files to read
+        required: True
+    notes:
+      - if read in variable context, the file can be interpreted as YAML if the content is valid to the parser.
+      - this lookup does not understand 'globing', use the fileglob lookup instead.
+"""
+
+EXAMPLES = """
+- debug: msg="the value of foo.txt is {{lookup('file', '/etc/foo.txt') }}"
+
+- name: display multiple file contents
+  debug: var=item
+  with_file:
+    - "/path/to/foo.txt"
+    - "bar.txt"  # will be looked in files/ dir relative to play or in role
+    - "/path/to/biz.txt"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - content of file(s)
+"""
+
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils._text import to_text

lib/ansible/plugins/lookup/fileglob.py

@@ -1,22 +1,45 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: fileglob
+    author: Michael DeHaan <michael.dehaan@gmail.com>
+    version_added: "1.4"
+    short_description: list files matching a pattern
+    description:
+        - Matches all files in a single directory, non-recursively, that match a pattern.
+          It calls Python's "glob" library.
+    options:
+      _terms:
+        description: path(s) of files to read
+        required: True
+    notes:
+      - Patterns ore only supported on files, not directory/paths.
+"""
+
+EXAMPLES = """
+- name: display content of all .txt files in dir
+  debug: msg={{lookup('fileglob', '/my/path/*.txt')}}
+
+- name: Copy each file over that matches the given pattern
+  copy:
+    src: "{{ item }}"
+    dest: "/etc/fooapp/"
+    owner: "root"
+    mode: 0600
+  with_fileglob:
+    - "/playbooks/files/fooapp/*"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - content of file(s)
+"""
+
 import os
 import glob
 

lib/ansible/plugins/lookup/filetree.py

@@ -1,22 +1,90 @@
 # (c) 2016 Dag Wieers <dag@wieers.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: filetree
+    author: Dag Wieers (@dagwieers) <dag@wieers.com>
+    version_added: "2.4"
+    short_description: recursively match all files in a directory tree
+    description:
+        - This lookup enables you to template a complete tree of files on a target system while retaining permissions and ownership.
+        - Supports directories, files and symlinks, including SELinux and other file properties
+        - If you provide more than one path, it will implement a with_first_found logic, and will not process entries it already processed in previous paths.
+          This enables merging different trees in order of importance, or add role_vars to specific paths to influence different instances of the same role.
+    options:
+      _terms:
+        description: path(s) of files to read
+        required: True
+"""
+
+EXAMPLES = """
+- name: Create directories
+  file:
+    path: /web/{{ item.path }}
+    state: directory
+    mode: '{{ item.mode }}'
+  with_filetree: web/
+  when: item.state == 'directory'
+
+- name: Template files
+  template:
+    src: '{{ item.src }}'
+    dest: /web/{{ item.path }}
+    mode: '{{ item.mode }}'
+  with_filetree: web/
+  when: item.state == 'file'
+
+- name: Recreate symlinks
+  file:
+    src: '{{ item.src }}'
+    dest: /web/{{ item.path }}
+    state: link
+    force: yes
+    mode: '{{ item.mode }}'
+  with_filetree: web/
+  when: item.state == 'link'
+"""
+
+RETURN = """
+  _raw:
+    description: list of dictionaries with file information
+    contains:
+        src:
+          description: TODO
+        root:
+          description: allows filtering by original location
+        path:
+          description: contains the relative path to root
+        mode:
+          description: TODO
+        state:
+          description: TODO
+        owner:
+          description: TODO
+        group:
+          description: TODO
+        seuser:
+          description: TODO
+        serole:
+          description: TODO
+        setype:
+          description: TODO
+        selevel:
+          description: TODO
+        uid:
+          description: TODO
+        gid:
+          description: TODO
+        size:
+          description: TODO
+        mtime:
+          description: TODO
+        ctime:
+          description: TODO
+"""
 import os
 import pwd
 import grp

lib/ansible/plugins/lookup/first_found.py

@@ -1,123 +1,64 @@
 # (c) 2013, seth vidal <skvidal@fedoraproject.org> red hat, inc
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-# take a list of files and (optionally) a list of paths
+DOCUMENTATION = """
-# return the first existing file found in the paths
+    lookup: first_found
-# [file1, file2, file3], [path1, path2, path3]
+    author: Seth Vidal <skvidal@fedoraproject.org>
-# search order is:
+    version_added: historical
-# path1/file1
+    short_description: return first file found from list
-# path1/file2
+    description:
-# path1/file3
+      - this lookup checks a list of files and paths and returns the full path to the first combination found.
-# path2/file1
+    options:
-# path2/file2
+      _terms:
-# path2/file3
+        description: list of file names
-# path3/file1
+        required: True
-# path3/file2
+      paths:
-# path3/file3
+        description: list of paths in which to look for the files
+"""
 
-# first file found with os.path.exists() is returned
+EXAMPLES = """
-# no file matches raises ansibleerror
+- name: show first existin file
-# EXAMPLES
+  debug: var=item
-# - name: copy first existing file found to /some/file
+  with_first_found:
-#   action: copy src=$item dest=/some/file
+    - "/path/to/foo.txt"
-#   with_first_found:
+    - "bar.txt"  # will be looked in files/ dir relative to play or in role
-#    - files: foo ${inventory_hostname} bar
+    - "/path/to/biz.txt"
-#      paths: /tmp/production /tmp/staging
 
-# that will look for files in this order:
+- name: copy first existing file found to /some/file
-# /tmp/production/foo
+  copy: src={{item}} dest=/some/file
-#                 ${inventory_hostname}
+  with_first_found:
-#                 bar
+    - foo
-# /tmp/staging/foo
+    - "{{inventory_hostname}}
-#              ${inventory_hostname}
+    - bar
-#              bar
 
-# - name: copy first existing file found to /some/file
+- name: same copy but specific paths
-#   action: copy src=$item dest=/some/file
+  copy: src={{item}} dest=/some/file
-#   with_first_found:
+  with_first_found:
-#    - files: /some/place/foo ${inventory_hostname} /some/place/else
+    files:
+      - foo
+      - "{{inventory_hostname}}
+      - bar
+    paths:
+      - /tmp/production
+      - /tmp/staging
 
-#  that will look for files in this order:
+- name: INTERFACES | Create Ansible header for /etc/network/interfaces
-#  /some/place/foo
+  template:
-#  $relative_path/${inventory_hostname}
+    src: "{{ item }}"
-#  /some/place/else
+    dest: "/etc/foo.conf"
-
+  with_first_found:
-# example - including tasks:
+    - "{{ ansible_virtualization_type }}_foo.conf"
-# tasks:
+    - "default_foo.conf"
-# - include: $item
+"""
-#   with_first_found:
-#    - files: generic
-#      paths: tasks/staging tasks/production
-# this will include the tasks in the file generic where it is found first (staging or production)
-
-# example simple file lists
-# tasks:
-# - name: first found file
-#   action: copy src=$item dest=/etc/file.cfg
-#   with_first_found:
-#   - files: foo.${inventory_hostname} foo
-
-
-# example skipping if no matched files
-# First_found also offers the ability to control whether or not failing
-# to find a file returns an error or not
-#
-# - name: first found file - or skip
-#   action: copy src=$item dest=/etc/file.cfg
-#   with_first_found:
-#   - files: foo.${inventory_hostname}
-#     skip: true
-
-# example a role with default configuration and configuration per host
-# you can set multiple terms with their own files and paths to look through.
-# consider a role that sets some configuration per host falling back on a default config.
-#
-# - name: some configuration template
-#   template: src={{ item }} dest=/etc/file.cfg mode=0444 owner=root group=root
-#   with_first_found:
-#    - files:
-#       - ${inventory_hostname}/etc/file.cfg
-#      paths:
-#       - ../../../templates.overwrites
-#       - ../../../templates
-#    - files:
-#       - etc/file.cfg
-#      paths:
-#       - templates
-
-# the above will return an empty list if the files cannot be found at all
-# if skip is unspecificed or if it is set to false then it will return a list
-# error which can be caught bye ignore_errors: true for that action.
-
-# finally - if you want you can use it, in place to replace first_available_file:
-# you simply cannot use the - files, path or skip options. simply replace
-# first_available_file with with_first_found and leave the file listing in place
-#
-#
-# - name: with_first_found like first_available_file
-#   action: copy src=$item dest=/tmp/faftest
-#   with_first_found:
-#    - ../files/foo
-#    - ../files/bar
-#    - ../files/baz
-#   ignore_errors: true
 
+RETURN = """
+  _raw:
+    description:
+      - path to file found
+"""
 import os
 
 from jinja2.exceptions import UndefinedError

lib/ansible/plugins/lookup/flattened.py

@@ -1,22 +1,36 @@
 # (c) 2013, Serge van Ginderachter <serge@vanginderachter.be>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: flattened
+    author: Serge van Ginderachter <serge@vanginderachter.be>
+    version_added: "1.3"
+    short_description: return single list completely flattened
+    description:
+      - given one or more lists, this lookup will flatten any list elements found recursively until only 1 list is left.
+    options:
+      _terms:
+        description: lists to flatten
+        required: True
+    notes:
+      - unlike 'items' which only flattens 1 level, this plugin will continue to flatten until it cannot find lists anymore.
+      - aka highlander plugin, there can only be one (list).
+"""
+
+EXAMPLES = """
+- name: "'unnest' all elements into single list"
+  debug: msg="all in one list {{lookup('flatten', [1,2,3,[5,6]], [a,b,c], [[5,6,1,3], [34,a,b,c]])}}"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - flattened list
+    type: list
+"""
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import string_types
 from ansible.plugins.lookup import LookupBase

lib/ansible/plugins/lookup/hashi_vault.py

@@ -1,48 +1,70 @@
 # (c) 2015, Jonathan Davila <jdavila(at)ansible.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-#
-# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}}
-#
-# To authenticate with a username/password against the LDAP auth backend in Vault:
-#
-# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello:value auth_method=ldap mount_point=ldap username=myuser password=mypassword url=http://myvault:8200')}}
-#
-# The mount_point param defaults to ldap, so is only required if you have a custom mount point.
-#
-# To use a ssl Vault add verify param:
-#
-# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=https://myvault:8200 validate_certs=False')}}
-#
-# The validate_certs param posible values are: True or False. By default it's in True. If False no verify of ssl will be done.
-# To use ca certificate file you can specify the path as parameter cacert
-#
-# USAGE: {{ lookup('hashi_vault', 'secret=secret/hello:value token=xxxx-xxx-xxx url=https://myvault:8200 validate_certs=True cacert=/cacert/path/ca.pem')}}
-#
-# You can skip setting the url if you set the VAULT_ADDR environment variable
-# or if you want it to default to localhost:8200
-#
-# NOTE: Due to a current limitation in the HVAC library there won't
-# necessarily be an error if a bad endpoint is specified.
-#
-# Requires hvac library. Install with pip.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+  lookup: hashi_vault
+  author: Jonathan Davila <jdavila(at)ansible.com>
+  version_added: "2.0"
+  short_description: retrieve secrets from HasihCorp's vault
+  requirements:
+    - hvac (python library)
+  description:
+    - retrieve secrets from HasihCorp's vault
+  notes:
+    - Due to a current limitation in the HVAC library there won't necessarily be an error if a bad endpoint is specified.
+  options:
+    secret:
+      description: query you are making
+      required: True
+    token:
+      description: vault token
+      env:
+        - name: VAULT_TOKEN
+    url:
+      description: url to vault service
+      env:
+        - name: VAULT_ADDR
+      default: 'http://127.0.0.1:8200'
+    username:
+      description: authentication user name
+    password:
+      description: authentication password
+    auth_method:
+      description: authentication method used
+    mount_point:
+      description: vault mount point, only required if you have a custom mount point
+      default: ldap
+    cacert:
+      description: path to certificate to use for authentication
+    validate_certs:
+      description: controls verification and validation of SSL certificates, mostly you only want to turn off with self signed ones.
+      type: boolean
+      default: True
+"""
+
+EXAMPLES = """
+- debug: msg="{{ lookup('hashi_vault', 'secret=secret/hello:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200')}}"
+
+- name: Vault that requires authentication via ldap
+  debug: msg="{{ lookup('hashi_vault', 'secret=secret/hello:value auth_method=ldap mount_point=ldap username=myuser password=mypas url=http://myvault:8200')}}"
+
+- name: Using an ssl vault
+  debug: msg="{{ lookup('hashi_vault', 'secret=secret/hola:value token=c975b780-d1be-8016-866b-01d0f9b688a5 url=https://myvault:8200 validate_certs=False')}}"
+
+- name: using certificate auth
+  debug: msg="{{ lookup('hashi_vault', 'secret=secret/hi:value token=xxxx-xxx-xxx url=https://myvault:8200 validate_certs=True cacert=/cacert/path/ca.pem')}}"
+"""
+
+RETURN = """
+_raw:
+  description:
+    - secrets(s) requested
+"""
+
 import os
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/hiera.py

@@ -1,75 +1,67 @@
 # (c) 2017, Juan Manuel Parrilla <jparrill@redhat.com>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
+from __future__ import (absolute_import, division, print_function)
-# Ansible is free software: you can redistribute it and/or modify
+__metaclass__ = type
-# it under the terms of the GNU General Public License as published by
+
-# the Free Software Foundation, either version 3 of the License, or
+DOCUMENTATION = '''
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-'''
-DOCUMENTATION:
     author:
-        - Juan Manuel Parrilla (@jparrill)
+      - Juan Manuel Parrilla (@jparrill)
     lookup: hiera
     version_added: "2.4"
     short_description: get info from hiera data
+    requirements:
+      - hiera (command line utility)
     description:
-        - Retrieves data from an Puppetmaster node using
+        - Retrieves data from an Puppetmaster node using Hiera as ENC
-          Hiera as ENC
     options:
-        _hiera_key:
+      _hiera_key:
             description:
                 - The list of keys to lookup on the Puppetmaster
             type: list
             element_type: string
             required: True
-        _bin_file:
+      _bin_file:
             description:
                 - Binary file to execute Hiera
             default: '/usr/bin/hiera'
-            env_vars:
+            env:
                 - name: ANSIBLE_HIERA_BIN
-        _hierarchy_file:
+      _hierarchy_file:
             description:
                 - File that describes the hierarchy of Hiera
             default: '/etc/hiera.yaml'
-            env_vars:
+            env:
                 - name: ANSIBLE_HIERA_CFG
-EXAMPLES:
+      FIXME:
-    All this examples depends on hiera.yml  that describes the
+        description: incomplete options .. _terms? environment/fqdn?  ANSIBLE_HIERA_CFG, ANSIBLE_HIERA_BIN
-    hierarchy
+'''
 
-    - name: "a value from Hiera 'DB'"
+EXAMPLES = """
-      debug: msg={{ lookup('hiera', 'foo') }}
+# All this examples depends on hiera.yml  that describes the hierarchy
 
-    - name: "a value from a Hiera 'DB' on other environment"
+- name: "a value from Hiera 'DB'"
-      debug: msg={{ lookup('hiera', 'foo environment=production') }}
+  debug: msg={{ lookup('hiera', 'foo') }}
 
-    - name: "a value from a Hiera 'DB' for a concrete node"
+- name: "a value from a Hiera 'DB' on other environment"
-      debug: msg={{ lookup('hiera', 'foo fqdn=puppet01.localdomain') }}
+  debug: msg={{ lookup('hiera', 'foo environment=production') }}
-RETURN:
+
-    _list:
+- name: "a value from a Hiera 'DB' for a concrete node"
+  debug: msg={{ lookup('hiera', 'foo fqdn=puppet01.localdomain') }}
+"""
+
+RETURN = """
+    _raw:
         description:
             - a value associated with input key
         type: strings
-'''
+"""
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
 
 import os
 
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.cmd_functions import run_cmd
 
-
 ANSIBLE_HIERA_CFG = os.getenv('ANSIBLE_HIERA_CFG', '/etc/hiera.yaml')
 ANSIBLE_HIERA_BIN = os.getenv('ANSIBLE_HIERA_BIN', '/usr/bin/hiera')
 

lib/ansible/plugins/lookup/indexed_items.py

@@ -1,22 +1,38 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: indexed_items
+    author: Michael DeHaan <michael.dehaan@gmail.com>
+    version_added: "1.3"
+    short_description: rewrites lists to return 'indexed items'
+    description:
+      - use this lookup if you want to loop over an array and also get the numeric index of where you are in the array as you go
+      - any list given will be transformed with each resulting element having the it's previous position in item.0 and its value in item.1
+    options:
+      _terms:
+        description: list of items
+        required: True
+"""
+
+EXAMPLES = """
+- name: indexed loop demo
+  debug:
+    msg: "at array position {{ item.0 }} there is a value {{ item.1 }}"
+  with_indexed_items:
+    - "{{ some_list }}"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - list with each item.0 giving you the postiion and item.1 the value
+    type: list
+"""
+
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 

lib/ansible/plugins/lookup/ini.py

@@ -1,22 +1,65 @@
 # (c) 2015, Yannig Perre <yannig.perre(at)gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: ini
+    author: Yannig Perre <yannig.perre(at)gmail.com>
+    version_added: "2.0"
+    short_description: read data from a ini file
+    description:
+      - "The ini lookup reads the contents of a file in INI format C(key1=value1).
+        This plugin retrieve the value on the right side after the equal sign C('=') of a given section C([section])."
+      - "You can also read a property file which - in this case - does not contain section."
+    options:
+      _terms:
+        description: they key(s) too look up
+        required: True
+    type:
+      description: ini Type of the file. 'properties' refers to the Java properties files.
+      default: 'ini'
+      choices: ['ini', 'properties']
+    file:
+      description: Name of the file to load
+      default: ansible.ini
+    section:
+      default: global
+      description: section where to lookup for key.
+    re:
+      default: False
+      type: boolean
+      description:  Flag to indicate if the key supplied is a regexp.
+    encoding:
+      default: utf-8
+      description:  Text encoding to use.
+    default:
+      description: return value if the key is not in the ini file
+      default: ''
+"""
+
+EXAMPLES = """
+- debug: msg="User in integration is {{ lookup('ini', 'user section=integration file=users.ini') }}"
+
+- debug: msg="User in production  is {{ lookup('ini', 'user section=production  file=users.ini') }}"
+
+- debug: msg="user.name is {{ lookup('ini', 'user.name type=properties file=user.properties') }}"
+
+- debug:
+    msg: "{{ item }}"
+  with_ini:
+    - value[1-2]
+    - section: section1
+    - file: "lookup.ini"
+    - re: true
+"""
+
+RETURN = """
+_raw:
+  description:
+    - value(s) of the key(s) in the ini file
+"""
 import os
 import re
 from collections import MutableSequence

lib/ansible/plugins/lookup/inventory_hostnames.py

@@ -1,24 +1,39 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
 # (c) 2013, Steven Dossett <sdossett@panath.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: inventory_hostnames
+    author:
+      - Michael DeHaan <michael.dehaan@gmail.com>
+      - Steven Dossett <sdossett@panath.com>
+    version_added: "1.3"
+    short_description: list of inventory hosts matching a host pattern
+    description:
+      - "This lookup understands 'host patterns' as used bye the `hosts:` keyword in plays
+        and can return a list of matching hosts from inventory"
+    notes:
+      - this is only worth for 'hostname patterns' it is easier to loop over the group/group_names variables otherwise.
+"""
+
+EXAMPLES = """
+- name: show all the hosts matching the pattern, ie all but the group www
+  debug:
+    msg: "{{ item }}"
+  with_inventory_hostnames:
+    - all:!www
+"""
+
+RETURN = """
+ _hostnames:
+    description: list of hostnames that matched the host pattern in inventory
+    type: list
+"""
+
 from ansible.inventory.manager import split_host_pattern, order_patterns
 from ansible.plugins.lookup import LookupBase
 

lib/ansible/plugins/lookup/items.py

@@ -1,22 +1,66 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: items
+    author: Michael DeHaan <michael.dehaan@gmail.com>
+    version_added: historical
+    short_description: list of items
+    description:
+      - this looup returns a list of items given to it, if any of the top level items is also a list it will flatten it, but it will not recurse
+    notes:
+      - this is the standard lookup used for loops in most examples
+      - check out the 'flattened' lookup for recursive flattening
+      - if you dont want flattening nor any other tranformation look at the 'list' lookup.
+    options:
+      _terms:
+        description: list of items
+        required: True
+"""
+
+EXAMPLES = """
+- name: "loop through list"
+  debug: msg="An item: {{item}}"
+  with_items:
+    - 1
+    - 2
+    - 3
+
+- name: add several users
+  user:
+    name: "{{ item }}"
+    groups: "wheel"
+    state: present
+  with_items:
+     - testuser1
+     - testuser2
+
+- name: "loop through list from a variable"
+  debug: msg="An item: {{item}}"
+  with_items: "{{ somelist }}"
+
+- name: more complex items to add several users
+  user:
+    name: "{{ item.name }}"
+    uid: "{{ item.uid }}"
+    groups: "{{ item.groups }}"
+    state: present
+  with_items:
+     - { name: testuser1, uid: 1002, groups: "wheel, staff" }
+     - { name: testuser2, uid: 1003, groups: staff }
+
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - once flattened list
+    type: list
+"""
+
 from ansible.plugins.lookup import LookupBase
 
 

lib/ansible/plugins/lookup/keyring.py

@@ -1,41 +1,38 @@
 # (c) 2016, Samuel Boucher <boucher.samuel.c@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
-'''
-Lookup plugin to grab secrets from the OS keyring.
-========================================================================================
 
-Warning the secret will be output to the screen
+DOCUMENTATION = """
+    lookup: keyring
+    author:
+      - Samuel Boucher <boucher.samuel.c@gmail.com>
+    version_added: "2.3"
+    requirements:
+      - keyring (python library)
+    short_description: grab secrets from the OS keyring
+    description:
+      - Allows you to access data stored in the OS provided keyring/keychain.
+"""
 
-Example:
+EXAMPLES = """
----
+- name : output secrets to screen (BAD IDEA)
-- hosts: localhost
+  debug:
-  tasks:
+    msg: "Password: {{item}}"
-    - name : test
+  with_keyring:
-      debug:
+    - 'servicename username'
-        msg: "Password: {{item}}"
-      with_keyring:
-        - 'servicename username'
 
-ansible localhost -m debug -a "msg=\"{{item}}\" with_keyring= 'servicename username'"
+- name: access mysql with password from keyring
+  mysql_db: login_password={{lookup('keyring','mysql joe')}} login_user=joe
+"""
+
+RETURN = """
+  _raw:
+    description: secrets stored
+"""
 
-'''
 HAS_KEYRING = True
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/lastpass.py

@@ -1,22 +1,38 @@
 # (c) 2016, Andrew Zenk <azenk@umn.edu>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: lastpass
+    author:
+      -  Andrew Zenk <azenk@umn.edu>
+    version_added: "2.3"
+    requirements:
+      - lpass (command line utility)
+      - must have already logged into lastpass
+    short_description: fetch data from lastpass
+    description:
+      - use the lpass command line utility to fetch specific fields from lastpass
+    options:
+      _terms:
+        description: key from which you want to retrieve the field
+        required: True
+      field:
+        description: field to return from lastpass
+        default: 'password'
+"""
+
+EXAMPLES = """
+# need one
+"""
+
+RETURN = """
+  _raw:
+    description: secrets stored
+"""
+
 from subprocess import Popen, PIPE
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/lines.py

@@ -1,23 +1,44 @@
 # (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: file
+    author: Daniel Hokka Zakrisson <daniel@hozac.com>
+    version_added: "0.9"
+    short_description: read lines from command
+    description:
+      - Run a commandi or more and split the output into lines returning them as a list
+    options:
+      _terms:
+        description: command(s) to run
+        required: True
+    notes:
+      - Like all lookups this runs on the Ansible controller and is unaffected by other keywords, such as become,
+        so if you need to different permissions you must change the command or run Ansible as another user.
+      - Alternatively you can use a shell/command task that runs against localhost and registers the result.
+"""
+
+EXAMPLES = """
+- name: we could use file direclty, but this shows output from command
+  debug: msg="{{ item }} is a line running cat on /etc/motd"
+  with_lines: cat /etc/motd
+
+- name: More useful example of looping over a command result
+  shell: "/usr/bin/frobnicate {{ item }}"
+  with_lines:
+    - "/usr/bin/frobnications_per_host --param {{ inventory_hostname }}"
+"""
+
+RETURN = """
+  _list:
+    description:
+      - lines of stdout from command
+"""
+
 import subprocess
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase

lib/ansible/plugins/lookup/list.py

@@ -1,24 +1,32 @@
-# (c) 2012-15, Ansible, Inc
+# (c) 2012-17 Ansible Project
-#
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 # Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: list
+    author: Ansible core team
+    version_added: "2.0"
+    short_description: simply returns what it is given.
+    description:
+      - this is mostly a noop, to be used as a with_list loop when you dont want the content transformed in any way.
+"""
+
+EXAMPLES = """
+- name: unlike with_items you will get 3 items from this loop, the 2nd one being a list
+  debug: var=item
+  with_list:
+    - 1
+    - [2,3]
+    - 4
+"""
+
+RETURN = """
+  _list:
+    description: basically the same as you fed in
+"""
 from ansible.plugins.lookup import LookupBase
 
 

lib/ansible/plugins/lookup/mongodb.py

@@ -68,7 +68,7 @@ DOCUMENTATION = '''
     notes:
         - "Please check https://api.mongodb.org/python/current/api/pymongo/collection.html?highlight=find#pymongo.collection.Collection.find for more detais."
     requirements:
-        - pymongo >= 2.4
+        - pymongo >= 2.4 (python library)
 '''
 
 EXAMPLES = '''

lib/ansible/plugins/lookup/nested.py

@@ -1,22 +1,52 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: nested
+    version_added: "1.1"
+    short_description: composes a list with nested elements of other lists
+    description:
+        - Takes the input lists and returns a list with elements that are lists composed of the elements of the input lists
+    options:
+      _raw:
+         description:
+           - a set of lists
+         required: True
+"""
+
+EXAMPLES = """
+- name: give users access to multiple databases
+  mysql_user:
+    name: "{{ item[0] }}"
+    priv: "{{ item[1] }}.*:ALL"
+    append_privs: yes
+    password: "foo"
+  with_nested:
+    - [ 'alice', 'bob' ]
+    - [ 'clientdb', 'employeedb', 'providerdb' ]
+As with the case of 'with_items' above, you can use previously defined variables.:
+
+- name: here, 'users' contains the above list of employees
+  mysql_user:
+    name: "{{ item[0] }}"
+    priv: "{{ item[1] }}.*:ALL"
+    append_privs: yes
+    password: "foo"
+  with_nested:
+    - "{{ users }}"
+    - [ 'clientdb', 'employeedb', 'providerdb' ]
+"""
+
+RETURN = """
+  _list:
+    description:
+      - A list composed of lists paring the elements of the input lists
+    type: list
+"""
+
 from jinja2.exceptions import UndefinedError
 
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable

lib/ansible/plugins/lookup/password.py

@@ -1,24 +1,94 @@
 # (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
 # (c) 2013, Javier Candeira <javier@candeira.com>
 # (c) 2013, Maykel Moya <mmoya@speedyrails.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: password
+    version_added: "1.1"
+    author:
+      - Daniel Hokka Zakrisson <daniel@hozac.com>
+      - Javier Candeira <javier@candeira.com>
+      - Maykel Moya <mmoya@speedyrails.com>
+    short_description: retrieve or generate a random password, stored in a file
+    description:
+      -  generates a random plaintext password and stores it in a file at a given filepath.
+      - 'If the file exists previously, it will retrieve its contents, behaving just like with_file.
+        Usage of variables like C("{{ inventory_hostname }}") in the filepath can be used to set up random passwords per host
+        (which simplifies password management in C('host_vars') variables).'
+      - A special case is using /dev/null as a path. The password lookup will generate a new random password each time,
+        but will not write it to /dev/null. This can be used when you need a password without storing it on the controller.
+    options:
+      _terms:
+         description:
+           - path to the file that stores/will store the passwords
+         required: True
+      encrypt:
+        description:
+           - Whether the user requests that this password is returned encrypted or in plain text.
+           - Note that the password is always stored as plain text.
+           - Encrypt also forces saving the salt value for idempotence.
+        type: boolean
+        default: True
+      chars:
+        version_added: "1.4"
+        description:
+          - Define comma separeted list of names that compose a custom character set in the generated passwords.
+          - 'By default generated passwords contain a random mix of upper and lowercase ASCII letters, the numbers 0-9 and punctuation (". , : - _").'
+          - "They can be either parts of Python's string module attributes (ascii_letters,digits, etc) or are used literally ( :, -)."
+          - "To enter comma use two commas ',,' somewhere - preferably at the end. Quotes and double quotes are not supported."
+        type: string
+      length:
+        description: The length of the generated password.
+        default: 20
+        type: integer
+    notes:
+      - A great alternative to the password lookup plugin,
+        if you don't need to generate random passwords on a per-host basis,
+        would be to use Using Vault in playbooks.
+        Read the documentation there and consider using it first,
+        it will be more desirable for most applications.
+      - If the file already exists, no data will be written to it.
+        If the file has contents, those contents will be read in as the password.
+        Empty files cause the password to return as an empty string.
+      - 'As all lookups, this runs on the Ansible host as the user running the playbook, and "become" does not apply,
+        the target file must be readable by the playbook user, or, if it does not exist,
+        the playbook user must have sufficient privileges to create it.
+        (So, for example, attempts to write into areas such as /etc will fail unless the entire playbook is being run as root).'
+"""
+
+EXAMPLES = """
+- name: create a mysql user with a random password
+  mysql_user:
+    name: "{{ client }}"
+    password: "{{ lookup('password', 'credentials/' + client + '/' + tier + '/' + role + '/mysqlpassword length=15') }}"
+    priv: "{{ client }}_{{ tier }}_{{ role }}.*:ALL"
+
+- name: create a mysql user with a random password using only ascii letters
+   mysql_user: name={{ client }} password="{{ lookup('password', '/tmp/passwordfile chars=ascii_letters') }}" priv={{ client }}_{{ tier }}_{{ role }}.*:ALL
+
+- name: create a mysql user with a random password using only digits
+  mysql_user:
+    name: "{{ client }}"
+    password: "{{ lookup('password', '/tmp/passwordfile chars=digits') }}"
+    priv: "{{ client }}_{{ tier }}_{{ role }}.*:ALL"
+
+- name: create a mysql user with a random password using many different char sets
+  mysql_user:
+    name: "{{ client }}"
+    password" "{{ lookup('password', '/tmp/passwordfile chars=ascii_letters,digits,hexdigits,punctuation') }}"
+    priv: "{{ client }}_{{ tier }}_{{ role }}.*:ALL"
+"""
+
+RETURN = """
+_raw:
+  description:
+    - a password
+"""
+
 import os
 import string
 

lib/ansible/plugins/lookup/passwordstore.py

@@ -1,22 +1,82 @@
 # (c) 2017, Patrick Deelman <patrick@patrickdeelman.nl>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+
+DOCUMENTATION = """
+    lookup: passwordstore
+    version_added: "2.3"
+    author:
+      - Patrick Deelman <patrick@patrickdeelman.nl>
+    short_description: manage passwords with passwordstore.org's pass utility
+    description:
+      - Enables Ansible to retrieve, create or update passwords from the passwordstore.org pass utility.
+        It also retrieves YAML style keys stored as multilines in the passwordfile.
+    options:
+      _terms:
+        descriptoin: query key
+        required: True
+      passwordstore:
+        description: location of the password store
+        default: '~/.password-store'
+      directory:
+        description: directory of the password store
+        default: null
+        env:
+          - name: PASSWORD_STORE_DIR
+      create:
+        description: flag to create the password
+        type: boolean
+        default: False
+      overwrite:
+        description: flag to overwrite the password
+        type: boolean
+        default: False
+      returnall:
+        description: flag to return all the contents of the password store
+        type: boolean
+        default: False
+      subkey:
+        description: subkey to return
+        default: password
+      userpass:
+        description: user password
+      length:
+        description: password length
+        type: integer
+        default: 16
+"""
+EXAMPLES = """
+# Debug is used for examples, BAD IDEA to show passwords on screen
+- name: Basic lookup. Fails if example/test doesn't exist
+  debug: msg="{{ lookup('passwordstore', 'example/test')}}"
+
+- name: Create pass with random 16 character password. If password exists just give the password
+  debug: var=mypassword
+  vars:
+    mypassword: "{{ lookup('passwordstore', 'example/test create=true')}}"
+
+- name: Different size password
+  debug: msg="{{ lookup('passwordstore', 'example/test create=true length=42')}}"
+
+- name: Create password and overwrite the password if it exists. As a bonus, this module includes the old password inside the pass file
+  debug: msg="{{ lookup('passwordstore', 'example/test create=true overwrite=true')}}"
+
+- name: Return the value for user in the KV pair user, username
+  debug: msg="{{ lookup('passwordstore', 'example/test subkey=user')}}"
+
+- name: Return the entire password file content
+  set_fact: passfilecontent="{{ lookup('passwordstore', 'example/test returnall=true')}}"
+"""
+
+RETURN = """
+_raw:
+  description:
+    - a password
+"""
+
 import os
 import subprocess
 import time

lib/ansible/plugins/lookup/pipe.py

@@ -1,22 +1,40 @@
 # (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: pipe
+    author: Daniel Hokka Zakrisson <daniel@hozac.com>
+    version_added: "0.9"
+    short_description: read output from a command
+    description:
+      - Run a command and return the output
+    options:
+      _terms:
+        description: command(s) to run
+        required: True
+    notes:
+      - Like all lookups this runs on the Ansible controller and is unaffected by other keywords, such as become,
+        so if you need to different permissions you must change the command or run Ansible as another user.
+      - Alternatively you can use a shell/command task that runs against localhost and registers the result.
+"""
+
+EXAMPLES = """
+- name: raw result of running date command"
+  debug: msg="{{ lookup('pipe','date') }}"
+
+- name: Always use quote filter to make sure your variables are safe to use with shell
+  debug: msg="{{ lookup('pipe','getent ' + myuser|quote ) }}"
+"""
+
+RETURN = """
+  _string:
+    description:
+      - stdout from command
+"""
+
 import subprocess
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/random_choice.py

@@ -1,38 +1,42 @@
 # (c) 2013, Michael DeHaan <michael.dehaan@gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: random_choice
+    author: Michael DeHaan <michael.dehaan@gmail.com>
+    version_added: "1.1"
+    short_description: return random element from list
+    description:
+      - The 'random_choice' feature can be used to pick something at random. While it's not a load balancer (there are modules for those),
+        it can somewhat be used as a poor man's load balancer in a MacGyver like situation.
+      - At a more basic level, they can be used to add chaos and excitement to otherwise predictable automation environments.
+"""
+
+EXAMPLES = """
+- name: Magic 8 ball for MUDs
+  debug:
+    msg: "{{ item }}"
+  with_random_choice:
+     - "go through the door"
+     - "drink from the goblet"
+     - "press the red button"
+     - "do nothing"
+"""
+
+RETURN = """
+  _raw:
+    description:
+      - random item
+"""
 import random
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_native
 from ansible.plugins.lookup import LookupBase
 
-# useful for introducing chaos ... or just somewhat reasonably fair selection
-# amongst available mirrors
-#
-#    tasks:
-#        - debug: msg=$item
-#          with_random_choice:
-#             - one
-#             - two
-#             - three
-
 
 class LookupModule(LookupBase):
 

lib/ansible/plugins/lookup/redis_kv.py

@@ -1,28 +1,45 @@
 # (c) 2012, Jan-Piet Mens <jpmens(at)gmail.com>
-#
+# (c) 2017 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: redis_kv
+    author: Jan-Piet Mens <jpmens(at)gmail.com>
+    version_added: "0.9"
+    short_description: fetch data from Redis
+    description:
+      - this looup returns a list of items given to it, if any of the top level items is also a list it will flatten it, but it will not recurse
+    requirements:
+      - redis (python library https://github.com/andymccurdy/redis-py/)
+    options:
+      _terms:
+        description: Two element comma separated strings composed of url of the Redis server and key to query
+        options:
+          _url:
+            description: location of redis host in url format
+            default: 'redis://localhost:6379'
+          _key:
+            description: key to query
+            required: True
+"""
+
+EXAMPLES = """
+- name: query redis for somekey
+  debug: msg="{{ lookup('redis_kv', 'redis://localhost:6379,somekey') }} is value in Redis for somekey"
+"""
+
+RETURN = """
+_raw:
+  description: values stored in Redis
+"""
 import os
 import re
 
 HAVE_REDIS = False
 try:
-    import redis        # https://github.com/andymccurdy/redis-py/
+    import redis
     HAVE_REDIS = True
 except ImportError:
     pass

lib/ansible/plugins/lookup/sequence.py

@@ -1,22 +1,70 @@
 # (c) 2013, Jayson Vantuyl <jayson@aggressive.ly>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: sequence
+    author: Jayson Vantuyl <jayson@aggressive.ly>
+    version_added: "1.0"
+    short_description: generate a list based on a number sequence
+    description:
+      - generates a sequence of items. You can specify a start value, an end value, an optional "stride" value that specifies the number of steps
+        to increment the sequence, and an optional printf-style format string.
+      - 'Arguments can be specified as key=value pair strings or as a shortcut form of the arguments string is also accepted: [start-]end[/stride][:format].'
+      - 'Numerical values can be specified in decimal, hexadecimal (0x3f8) or octal (0600).'
+      - Starting at version 1.9.2, negative strides are allowed.
+    options:
+      start:
+        description: number at which to start the sequence
+        default: 0
+        type: number
+      end:
+        description: number at which to end the sequence, dont use this with count
+        type: number
+        default: 0
+      count:
+        description: number of elements in the sequence, this is not to be used with end
+        type: number
+        default: 0
+      stride:
+        description: increments between sequence numbers, the default is 1 unless the end is less than the start, then it is -1.
+        type: number
+      format:
+        description: return a string with the generated number formated in
+"""
+
+EXAMPLES = """
+- name: create some test users
+  user:
+    name: "{{ item }}"
+    state: present
+    groups: "evens"
+  with_sequence: start=0 end=32 format=testuser%02x
+
+- name: create a series of directories with even numbers for some reason
+  file:
+    dest: "/var/stuff/{{ item }}"
+    state: directory
+  with_sequence: start=4 end=16 stride=2
+
+- name: a simpler way to use the sequence plugin create 4 groups
+  group:
+    name: "group{{ item }}"
+    state: present
+  with_sequence: count=4
+
+- name: the final countdown
+  debug: msg={{item}} seconds to detonation
+  with_sequence: end=0 start=10
+"""
+
+RETURN = """
+  _list:
+    description: generated sequence of numbers or strings
+"""
+
 from re import compile as re_compile, IGNORECASE
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/shelvefile.py

@@ -1,22 +1,36 @@
 # (c) 2015, Alejandro Guirao <lekumberri@gmail.com>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: shelvefile
+    author: Alejandro Guirao <lekumberri@gmail.com>
+    version_added: "2.0"
+    short_description: read keys from Python shelve file
+    description:
+    optoins:
+      _terms:
+        description: sets of key value pairs of parameters
+        options:
+          key:
+            description: key to query
+            required: True
+          file:
+            description: path to shelve file
+            required: True
+"""
+
+EXAMPLES = """
+- name: retrieve a string value corresponding to a key inside a Python shelve file
+  debug: msg="{{ lookup('shelvefile', 'file=path_to_some_shelve_file.db key=key_to_retrieve') }}
+"""
+
+RETURN = """
+_list:
+  description: value(s) of key(s) in shelve file(s)
+"""
 import shelve
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/subelements.py

@@ -1,22 +1,87 @@
 # (c) 2013, Serge van Ginderachter <serge@vanginderachter.be>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: subelements
+    author: Serge van Ginderachter <serge@vanginderachter.be>
+    version_added: "1.4"
+    short_description: traverse nested key from a list of dictionaries
+    description:
+      - Subelements walks a list of hashes (aka dictionaries) and then traverses a list with a given (nested sub-)key inside of those records.
+    options:
+      _terms:
+         description: tuple of list of dictionaries and dictionary key to extract
+         required: True
+      skip_missing:
+        default: False
+        description:
+          - If set to True, the lookup plugin will skip the lists items that do not contain the given subkey.
+            If False, the plugin will yield an error and complain about the missing subkey.
+"""
+
+EXAMPLES = """
+- name: show var structure as it is needed for example to make sense
+  hosts: all
+  vars:
+    users:
+      - name: alice
+        authorized:
+          - /tmp/alice/onekey.pub
+          - /tmp/alice/twokey.pub
+        mysql:
+            password: mysql-password
+            hosts:
+              - "%"
+              - "127.0.0.1"
+              - "::1"
+              - "localhost"
+            privs:
+              - "*.*:SELECT"
+              - "DB1.*:ALL"
+        groups:
+          - wheel
+      - name: bob
+        authorized:
+          - /tmp/bob/id_rsa.pub
+        mysql:
+            password: other-mysql-password
+            hosts:
+              - "db1"
+            privs:
+              - "*.*:SELECT"
+              - "DB2.*:ALL"
+  tasks:
+    - name: Set authorized ssh key, extracting just that data from 'users'
+      authorized_key:
+        user: "{{ item.0.name }}"
+        key: "{{ lookup('file', item.1) }}"
+      with_subelements:
+         - "{{ users }}"
+         - authorized
+
+    - name: Setup MySQL users, given the mysql hosts and privs subkey lists
+      mysql_user:
+        name: "{{ item.0.name }}"
+        password: "{{ item.0.mysql.password }}"
+        host: "{{ item.1 }}"
+        priv: "{{ item.0.mysql.privs | join('/') }}"
+      with_subelements:
+        - "{{ users }}"
+        - mysql.hosts
+
+    - name: list groups for user that have them, dont error if they don't
+      debug: var=item
+      with_list: "{{lookup('subelements', users, 'groups', 'skip_missing=True')}}"
+"""
+
+RETURN = """
+_list:
+  description: list of subelements extracted
+"""
+
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import string_types
 from ansible.module_utils.parsing.convert_bool import boolean

lib/ansible/plugins/lookup/template.py

@@ -1,22 +1,34 @@
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: template
+    author: Michael DeHaan <michael.dehaan@gmail.com>
+    version_added: "0.9"
+    short_description: retrieve contents of file after templating with Jinja2
+    description:
+      - this is mostly a noop, to be used as a with_list loop when you dont want the content transformed in any way.
+    options:
+      _terms:
+        description: list of files to template
+"""
+
+EXAMPLES = """
+- name: show templating results
+  debug: msg="{{ lookup('template', './some_template.j2') }}
+"""
+
+RETURN = """
+_raw:
+   description: file(s) content after templating
+"""
+
 import os
 
 from ansible.errors import AnsibleError

lib/ansible/plugins/lookup/together.py

@@ -1,22 +1,38 @@
 # (c) 2013, Bradley Young <young.bradley@gmail.com>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: toghether
+    author:  Bradley Young <young.bradley@gmail.com>
+    version_added: '1.3'
+    short_description: merges lists into syncronized list
+    description:
+      - Creates a list with the iterated elements of the supplied lists
+      - "To clarify with an example, [ 'a', 'b' ] and [ 1, 2 ] turn into [ ('a',1), ('b', 2) ]"
+      - This is basicaly the same as the 'zip_longest' filter and Python function
+      - Any 'unbalanced' elements will be substituted with 'None'
+    options:
+      _terms:
+        description: list of lists to merge
+        required: True
+"""
+
+EXAMPLES = """
+- name: item.0 returns from the 'a' list, item.1 returns from the '1' list
+  debug:
+    msg: "{{ item.0 }} and {{ item.1 }}"
+  with_together:
+    - ['a', 'b', 'c', 'd']
+    - [1, 2, 3, 4]
+"""
+
+RETURN = """
+  _list:
+    description: syncronized list
+"""
 from ansible.errors import AnsibleError
 from ansible.module_utils.six.moves import zip_longest
 from ansible.plugins.lookup import LookupBase

lib/ansible/plugins/lookup/url.py

@@ -1,22 +1,42 @@
 # (c) 2015, Brian Coca <bcoca@ansible.com>
-#
+# (c) 2012-17 Ansible Project
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+    lookup: url
+    author: Brian Coca <bcoca@ansible.com>
+    version_added: "1.9"
+    short_description: return contents from URL
+    description:
+      _terms:
+        description: urls to query
+      validate_certs:
+        description: Flag to control SSL certificate validation
+        type: boolean
+        default: True
+      split_lines:
+        description: Flag to control if content is returned as a list of lines or as a single text blob
+        type: boolean
+        default: True
+      use_proxy:
+        description: Flag to control if the lookup will observe HTTP proxy environment variables when present.
+        type: boolean
+        default: True
+"""
+
+EXAMPLES = """
+- name: url lookup splits lines by default
+  debug: msg="{{item}}"
+  with_url:
+    - 'https://github.com/gremlin.keys'
+"""
+
+RETURN = """
+  _list:
+    description: list of list of lines or content of url(s)
+"""
 
 from ansible.errors import AnsibleError
 from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError