Add integration tests for add_collection_to_versions_and_dates(), and extend ansible-doc tests (#73601)

* Add add_collection_to_versions_and_dates integration tests.

* Add doc fragment tests with fragments from other collections.

* Remove trailing whitespace (Python 2's json.dump inserts some).

* Use ' ' instead of '\s'.

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol/plugins/cache/notjsonfile.py

@@ -11,6 +11,7 @@ DOCUMENTATION = '''
     description:
         - This cache uses JSON formatted, per host, files saved to the filesystem.
     author: Ansible Core (@ansible-core)
+    version_added: 0.7.0
     options:
       _uri:
         required: True
@@ -18,16 +19,26 @@ DOCUMENTATION = '''
           - Path in which the cache plugin will save the JSON files
         env:
           - name: ANSIBLE_CACHE_PLUGIN_CONNECTION
+            version_added: 1.2.0
         ini:
           - key: fact_caching_connection
             section: defaults
+            deprecated:
+              alternative: none
+              why: Test deprecation
+              version: '2.0.0'
       _prefix:
         description: User defined prefix to use when creating the JSON files
         env:
           - name: ANSIBLE_CACHE_PLUGIN_PREFIX
+            version_added: 1.1.0
         ini:
           - key: fact_caching_prefix
             section: defaults
+        deprecated:
+          alternative: none
+          why: Another test deprecation
+          removed_at_date: '2050-01-01'
       _timeout:
         default: 86400
         description: Expiration timeout for the cache plugin data
@@ -36,7 +47,16 @@ DOCUMENTATION = '''
         ini:
           - key: fact_caching_timeout
             section: defaults
+        vars:
+          - name: notsjonfile_fact_caching_timeout
+            version_added: 1.5.0
+            deprecated:
+              alternative: do not use a variable
+              why: Test deprecation
+              version: '3.0.0'
         type: integer
+    extends_documentation_fragment:
+        - testns.testcol2.plugin
 '''
 
 from ansible.plugins.cache import BaseFileCacheModule

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol/plugins/lookup/noop.py

@@ -12,6 +12,12 @@ DOCUMENTATION = """
     short_description: returns input
     description:
       - this is a noop
+    deprecated:
+        alternative: Use some other lookup
+        why: Test deprecation
+        removed_in: '3.0.0'
+    extends_documentation_fragment:
+        - testns.testcol2.version_added
 """
 
 EXAMPLES = """
@@ -22,6 +28,7 @@ EXAMPLES = """
 RETURN = """
   _list:
     description: input given
+    version_added: 1.0.0
 """
 
 from ansible.module_utils.common._collections_compat import Sequence

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol/plugins/modules/fakemodule.py

@@ -8,6 +8,7 @@ DOCUMENTATION = """
     short_desciption: fake module
     description:
         - this is a fake module
+    version_added: 1.0.0
     options:
         _notreal:
             description:  really not a real option

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol/plugins/modules/randommodule.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+---
+module: randommodule
+short_description: A random module
+description:
+    - A random module.
+author:
+    - Ansible Core Team
+version_added: 1.0.0
+deprecated:
+    alternative: Use some other module
+    why: Test deprecation
+    removed_in: '3.0.0'
+options:
+    test:
+        description: Some text.
+        type: str
+        version_added: 1.2.0
+    sub:
+        description: Suboptions.
+        type: dict
+        suboptions:
+            subtest:
+                description: A suboption.
+                type: int
+                version_added: 1.1.0
+        # The following is the wrong syntax, and should not get processed
+        # by add_collection_to_versions_and_dates()
+        options:
+            subtest2:
+                description: Another suboption.
+                type: float
+                version_added: 1.1.0
+        # The following is not supported in modules, and should not get processed
+        # by add_collection_to_versions_and_dates()
+        env:
+            - name: TEST_ENV
+              version_added: 1.0.0
+              deprecated:
+                alternative: none
+                why: Test deprecation
+                removed_in: '2.0.0'
+                version: '2.0.0'
+extends_documentation_fragment:
+    - testns.testcol2.module
+'''
+
+EXAMPLES = '''
+'''
+
+RETURN = '''
+z_last:
+    description: A last result.
+    type: str
+    returned: success
+    version_added: 1.3.0
+
+m_middle:
+    description:
+        - This should be in the middle.
+        - Has some more data
+    type: dict
+    returned: success and 1st of month
+    contains:
+        suboption:
+            description: A suboption.
+            type: str
+            choices: [ARF, BARN, c_without_capital_first_letter]
+            version_added: 1.4.0
+
+a_first:
+    description: A first result.
+    type: str
+    returned: success
+'''
+
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(),
+    )
+
+    module.exit_json()
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol/plugins/vars/noop_vars_plugin.py

@@ -15,6 +15,8 @@ DOCUMENTATION = '''
             section: testns.testcol.noop_vars_plugin
         env:
           - name: ANSIBLE_VARS_PLUGIN_STAGE
+    extends_documentation_fragment:
+        - testns.testcol2.deprecation
 '''
 
 from ansible.plugins.vars import BaseVarsPlugin

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol2/MANIFEST.json

@@ -0,0 +1,30 @@
+{
+ "collection_info": {
+  "description": null, 
+  "repository": "",
+  "tags": [], 
+  "dependencies": {}, 
+  "authors": [
+   "Ansible (https://ansible.com)"
+  ], 
+  "issues": "",
+  "name": "testcol2", 
+  "license": [
+   "GPL-3.0-or-later"
+  ], 
+  "documentation": "",
+  "namespace": "testns", 
+  "version": "1.2.0", 
+  "readme": "README.md", 
+  "license_file": "COPYING", 
+  "homepage": "",
+ }, 
+ "file_manifest_file": {
+  "format": 1, 
+  "ftype": "file", 
+  "chksum_sha256": "4c15a867ceba8ba1eaf2f4a58844bb5dbb82fec00645fc7eb74a3d31964900f6", 
+  "name": "FILES.json", 
+  "chksum_type": "sha256"
+ }, 
+ "format": 1
+}

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol2/plugins/doc_fragments/deprecation.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+    DOCUMENTATION = r'''
+options: {}
+deprecated:
+    alternative: Use some other module
+    why: Test deprecation
+    removed_in: '3.0.0'
+'''

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol2/plugins/doc_fragments/module.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+    DOCUMENTATION = r'''
+options:
+    testcol2option:
+        description:
+            - An option taken from testcol2
+        type: str
+        version_added: 1.0.0
+    testcol2option2:
+        description:
+            - Another option taken from testcol2
+        type: str
+'''

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol2/plugins/doc_fragments/plugin.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+    DOCUMENTATION = r'''
+options:
+    testcol2option:
+        description:
+            - A plugin option taken from testcol2
+        type: str
+        version_added: 1.0.0
+        ini:
+            - key: foo
+              section: bar
+              version_added: 1.1.0
+              deprecated:
+                alternative: none
+                why: Test deprecation
+                version: '3.0.0'
+        env:
+            - name: FOO_BAR
+              version_added: 1.2.0
+              deprecated:
+                alternative: none
+                why: Test deprecation
+                removed_at_date: 2020-01-31
+        vars:
+            - name: foobar
+              version_added: 1.3.0
+              deprecated:
+                alternative: none
+                why: Test deprecation
+                removed_at_date: 2040-12-31
+    testcol2depr:
+        description:
+            - A plugin option taken from testcol2 that is deprecated
+        type: str
+        deprecated:
+            alternative: none
+            why: Test option deprecation
+            version: '2.0.0'
+'''

test/integration/targets/ansible-doc/collections/ansible_collections/testns/testcol2/plugins/doc_fragments/version_added.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+    DOCUMENTATION = r'''
+options: {}
+version_added: 1.0.0
+'''

test/integration/targets/ansible-doc/fakemodule.output

@@ -12,3 +12,5 @@ OPTIONS (= is mandatory):
 AUTHOR: me
 
 SHORT_DESCIPTION: fake module
+
+VERSION_ADDED_COLLECTION: testns.testcol

test/integration/targets/ansible-doc/noop.output

@@ -0,0 +1,32 @@
+{
+    "testns.testcol.noop": {
+        "doc": {
+            "author": "Ansible core team",
+            "collection": "testns.testcol",
+            "deprecated": {
+                "alternative": "Use some other lookup",
+                "removed_from_collection": "testns.testcol",
+                "removed_in": "3.0.0",
+                "why": "Test deprecation"
+            },
+            "description": [
+                "this is a noop"
+            ],
+            "filename": "./collections/ansible_collections/testns/testcol/plugins/lookup/noop.py",
+            "lookup": "noop",
+            "options": {},
+            "short_description": "returns input",
+            "version_added": "1.0.0",
+            "version_added_collection": "testns.testcol2"
+        },
+        "examples": "\n- name: do nothing\n  debug: msg=\"{{ lookup('testns.testcol.noop', [1,2,3,4] }}\"\n",
+        "metadata": null,
+        "return": {
+            "_list": {
+                "description": "input given",
+                "version_added": "1.0.0",
+                "version_added_collection": "testns.testcol"
+            }
+        }
+    }
+}

test/integration/targets/ansible-doc/noop_vars_plugin.output

@@ -0,0 +1,42 @@
+{
+    "testns.testcol.noop_vars_plugin": {
+        "doc": {
+            "collection": "testns.testcol",
+            "deprecated": {
+                "alternative": "Use some other module",
+                "removed_from_collection": "testns.testcol2",
+                "removed_in": "3.0.0",
+                "why": "Test deprecation"
+            },
+            "description": "don't test loading host and group vars from a collection",
+            "filename": "./collections/ansible_collections/testns/testcol/plugins/vars/noop_vars_plugin.py",
+            "options": {
+                "stage": {
+                    "choices": [
+                        "all",
+                        "inventory",
+                        "task"
+                    ],
+                    "default": "all",
+                    "env": [
+                        {
+                            "name": "ANSIBLE_VARS_PLUGIN_STAGE"
+                        }
+                    ],
+                    "ini": [
+                        {
+                            "key": "stage",
+                            "section": "testns.testcol.noop_vars_plugin"
+                        }
+                    ],
+                    "type": "str"
+                }
+            },
+            "short_description": "Do NOT load host and group vars",
+            "vars": "noop_vars_plugin"
+        },
+        "examples": null,
+        "metadata": null,
+        "return": null
+    }
+}

test/integration/targets/ansible-doc/notjsonfile.output

@@ -0,0 +1,157 @@
+{
+    "testns.testcol.notjsonfile": {
+        "doc": {
+            "author": "Ansible Core (@ansible-core)",
+            "cache": "notjsonfile",
+            "collection": "testns.testcol",
+            "description": [
+                "This cache uses JSON formatted, per host, files saved to the filesystem."
+            ],
+            "filename": "./collections/ansible_collections/testns/testcol/plugins/cache/notjsonfile.py",
+            "options": {
+                "_prefix": {
+                    "deprecated": {
+                        "alternative": "none",
+                        "collection_name": "testns.testcol",
+                        "removed_at_date": "2050-01-01",
+                        "why": "Another test deprecation"
+                    },
+                    "description": "User defined prefix to use when creating the JSON files",
+                    "env": [
+                        {
+                            "name": "ANSIBLE_CACHE_PLUGIN_PREFIX",
+                            "version_added": "1.1.0",
+                            "version_added_collection": "testns.testcol"
+                        }
+                    ],
+                    "ini": [
+                        {
+                            "key": "fact_caching_prefix",
+                            "section": "defaults"
+                        }
+                    ]
+                },
+                "_timeout": {
+                    "default": 86400,
+                    "description": "Expiration timeout for the cache plugin data",
+                    "env": [
+                        {
+                            "name": "ANSIBLE_CACHE_PLUGIN_TIMEOUT"
+                        }
+                    ],
+                    "ini": [
+                        {
+                            "key": "fact_caching_timeout",
+                            "section": "defaults"
+                        }
+                    ],
+                    "type": "integer",
+                    "vars": [
+                        {
+                            "deprecated": {
+                                "alternative": "do not use a variable",
+                                "collection_name": "testns.testcol",
+                                "version": "3.0.0",
+                                "why": "Test deprecation"
+                            },
+                            "name": "notsjonfile_fact_caching_timeout",
+                            "version_added": "1.5.0",
+                            "version_added_collection": "testns.testcol"
+                        }
+                    ]
+                },
+                "_uri": {
+                    "description": [
+                        "Path in which the cache plugin will save the JSON files"
+                    ],
+                    "env": [
+                        {
+                            "name": "ANSIBLE_CACHE_PLUGIN_CONNECTION",
+                            "version_added": "1.2.0",
+                            "version_added_collection": "testns.testcol"
+                        }
+                    ],
+                    "ini": [
+                        {
+                            "deprecated": {
+                                "alternative": "none",
+                                "collection_name": "testns.testcol",
+                                "version": "2.0.0",
+                                "why": "Test deprecation"
+                            },
+                            "key": "fact_caching_connection",
+                            "section": "defaults"
+                        }
+                    ],
+                    "required": true
+                },
+                "testcol2depr": {
+                    "deprecated": {
+                        "alternative": "none",
+                        "collection_name": "testns.testcol2",
+                        "version": "2.0.0",
+                        "why": "Test option deprecation"
+                    },
+                    "description": [
+                        "A plugin option taken from testcol2 that is deprecated"
+                    ],
+                    "type": "str"
+                },
+                "testcol2option": {
+                    "description": [
+                        "A plugin option taken from testcol2"
+                    ],
+                    "env": [
+                        {
+                            "deprecated": {
+                                "alternative": "none",
+                                "collection_name": "testns.testcol2",
+                                "removed_at_date": "2020-01-31",
+                                "why": "Test deprecation"
+                            },
+                            "name": "FOO_BAR",
+                            "version_added": "1.2.0",
+                            "version_added_collection": "testns.testcol2"
+                        }
+                    ],
+                    "ini": [
+                        {
+                            "deprecated": {
+                                "alternative": "none",
+                                "collection_name": "testns.testcol2",
+                                "version": "3.0.0",
+                                "why": "Test deprecation"
+                            },
+                            "key": "foo",
+                            "section": "bar",
+                            "version_added": "1.1.0",
+                            "version_added_collection": "testns.testcol2"
+                        }
+                    ],
+                    "type": "str",
+                    "vars": [
+                        {
+                            "deprecated": {
+                                "alternative": "none",
+                                "collection_name": "testns.testcol2",
+                                "removed_at_date": "2040-12-31",
+                                "why": "Test deprecation"
+                            },
+                            "name": "foobar",
+                            "version_added": "1.3.0",
+                            "version_added_collection": "testns.testcol2"
+                        }
+                    ],
+                    "version_added": "1.0.0",
+                    "version_added_collection": "testns.testcol2"
+                }
+            },
+            "short_description": "JSON formatted files.",
+            "version_added": "0.7.0",
+            "version_added_collection": "testns.testcol"
+        },
+        "examples": null,
+        "metadata": null,
+        "return": null
+    }
+}

test/integration/targets/ansible-doc/randommodule.output

@@ -0,0 +1,115 @@
+{
+    "testns.testcol.randommodule": {
+        "doc": {
+            "author": [
+                "Ansible Core Team"
+            ],
+            "collection": "testns.testcol",
+            "deprecated": {
+                "alternative": "Use some other module",
+                "removed_from_collection": "testns.testcol",
+                "removed_in": "3.0.0",
+                "why": "Test deprecation"
+            },
+            "description": [
+                "A random module."
+            ],
+            "filename": "./collections/ansible_collections/testns/testcol/plugins/modules/randommodule.py",
+            "has_action": false,
+            "module": "randommodule",
+            "options": {
+                "sub": {
+                    "description": "Suboptions.",
+                    "env": [
+                        {
+                            "deprecated": {
+                                "alternative": "none",
+                                "removed_in": "2.0.0",
+                                "version": "2.0.0",
+                                "why": "Test deprecation"
+                            },
+                            "name": "TEST_ENV",
+                            "version_added": "1.0.0"
+                        }
+                    ],
+                    "options": {
+                        "subtest2": {
+                            "description": "Another suboption.",
+                            "type": "float",
+                            "version_added": "1.1.0"
+                        }
+                    },
+                    "suboptions": {
+                        "subtest": {
+                            "description": "A suboption.",
+                            "type": "int",
+                            "version_added": "1.1.0",
+                            "version_added_collection": "testns.testcol"
+                        }
+                    },
+                    "type": "dict"
+                },
+                "test": {
+                    "description": "Some text.",
+                    "type": "str",
+                    "version_added": "1.2.0",
+                    "version_added_collection": "testns.testcol"
+                },
+                "testcol2option": {
+                    "description": [
+                        "An option taken from testcol2"
+                    ],
+                    "type": "str",
+                    "version_added": "1.0.0",
+                    "version_added_collection": "testns.testcol2"
+                },
+                "testcol2option2": {
+                    "description": [
+                        "Another option taken from testcol2"
+                    ],
+                    "type": "str"
+                }
+            },
+            "short_description": "A random module",
+            "version_added": "1.0.0",
+            "version_added_collection": "testns.testcol"
+        },
+        "examples": "\n",
+        "metadata": null,
+        "return": {
+            "a_first": {
+                "description": "A first result.",
+                "returned": "success",
+                "type": "str"
+            },
+            "m_middle": {
+                "contains": {
+                    "suboption": {
+                        "choices": [
+                            "ARF",
+                            "BARN",
+                            "c_without_capital_first_letter"
+                        ],
+                        "description": "A suboption.",
+                        "type": "str",
+                        "version_added": "1.4.0",
+                        "version_added_collection": "testns.testcol"
+                    }
+                },
+                "description": [
+                    "This should be in the middle.",
+                    "Has some more data"
+                ],
+                "returned": "success and 1st of month",
+                "type": "dict"
+            },
+            "z_last": {
+                "description": "A last result.",
+                "returned": "success",
+                "type": "str",
+                "version_added": "1.3.0",
+                "version_added_collection": "testns.testcol"
+            }
+        }
+    }
+}

test/integration/targets/ansible-doc/runme.sh

@@ -76,3 +76,21 @@ expected_role_out="$(sed '1 s/\(^> TESTNS\.TESTCOL\.TESTROLE\).*(.*)$/\1/' fakec
 test "$current_role_out" == "$expected_role_out"
 
 )
+
+#### test add_collection_to_versions_and_dates()
+
+current_out="$(ansible-doc --json --playbook-dir ./ testns.testcol.randommodule | sed 's/ *$//' | sed 's/ *"filename": "[^"]*",$//')"
+expected_out="$(sed 's/ *"filename": "[^"]*",$//' randommodule.output)"
+test "$current_out" == "$expected_out"
+
+current_out="$(ansible-doc --json --playbook-dir ./ -t cache testns.testcol.notjsonfile | sed 's/ *$//' | sed 's/ *"filename": "[^"]*",$//')"
+expected_out="$(sed 's/ *"filename": "[^"]*",$//' notjsonfile.output)"
+test "$current_out" == "$expected_out"
+
+current_out="$(ansible-doc --json --playbook-dir ./ -t lookup testns.testcol.noop | sed 's/ *$//' | sed 's/ *"filename": "[^"]*",$//')"
+expected_out="$(sed 's/ *"filename": "[^"]*",$//' noop.output)"
+test "$current_out" == "$expected_out"
+
+current_out="$(ansible-doc --json --playbook-dir ./ -t vars testns.testcol.noop_vars_plugin | sed 's/ *$//' | sed 's/ *"filename": "[^"]*",$//')"
+expected_out="$(sed 's/ *"filename": "[^"]*",$//' noop_vars_plugin.output)"
+test "$current_out" == "$expected_out"