Fix junos resource modules group based config and minor updates (#61224)

* Fix junos resource modules group based config and minor updates

Fixes https://github.com/ansible/ansible/issues/61183

*  Add support to get inherited configuration for resource
   modules to handle group based configuration
*  Add task input check for merged, replaced and overridden
   states in junos resource modules
*  Integration test for group based configuration

* Fix CI test failures

* Fix test failures

lib/ansible/module_utils/network/junos/config/l2_interfaces/l2_interfaces.py

@@ -18,6 +18,7 @@ from ansible.module_utils.network.common.utils import to_list
 from ansible.module_utils.network.common.cfg.base import ConfigBase
 from ansible.module_utils.network.junos.junos import locked_config, load_config, commit_configuration, discard_changes, tostring
 from ansible.module_utils.network.junos.facts.facts import Facts
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.network.common.netconf import build_root_xml_node, build_child_xml_node, build_subtree
 
 
@@ -220,7 +221,7 @@ class L2_interfaces(ConfigBase):
                 <interfaces/>
             </configuration>
             """
-        data = self._connection.get_configuration(filter=config_filter)
+        data = get_resource_config(self._connection, config_filter=config_filter)
 
         if not l2_intf_obj:
             # delete l2 interfaces attribute from all the existing interface having l2 config

lib/ansible/module_utils/network/junos/config/lag_interfaces/lag_interfaces.py

@@ -17,6 +17,7 @@ from ansible.module_utils.network.common.cfg.base import ConfigBase
 from ansible.module_utils.network.common.utils import to_list
 from ansible.module_utils.network.junos.facts.facts import Facts
 from ansible.module_utils.network.junos.junos import locked_config, load_config, commit_configuration, discard_changes, tostring
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.network.common.netconf import build_root_xml_node, build_child_xml_node, build_subtree
 
 
@@ -167,7 +168,7 @@ class Lag_interfaces(ConfigBase):
                 <interfaces/>
             </configuration>
             """
-        data = self._connection.get_configuration(filter=config_filter)
+        data = get_resource_config(self._connection, config_filter=config_filter)
 
         for config in want:
             lag_name = config['name']

lib/ansible/module_utils/network/junos/facts/interfaces/interfaces.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.interfaces.interfaces import InterfacesArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -56,11 +57,11 @@ class InterfacesFacts(object):
 
         if not data:
             config_filter = """
-            <configuration>
+                <configuration>
-                <interfaces/>
+                    <interfaces/>
-            </configuration>
+                </configuration>
             """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/l2_interfaces/l2_interfaces.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.l2_interfaces.l2_interfaces import L2_interfacesArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -60,7 +61,7 @@ class L2_interfacesFacts(object):
                     <interfaces/>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/lacp/lacp.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.lacp.lacp import LacpArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -67,7 +68,7 @@ class LacpFacts(object):
                     </chassis>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/lacp_interfaces/lacp_interfaces.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.lacp_interfaces.lacp_interfaces import Lacp_interfacesArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -60,7 +61,7 @@ class Lacp_interfacesFacts(object):
                     <interfaces/>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/lag_interfaces/lag_interfaces.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.lag_interfaces.lag_interfaces import Lag_interfacesArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -60,7 +61,7 @@ class Lag_interfacesFacts(object):
                     <interfaces/>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/lldp_global/lldp_global.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.lldp_global.lldp_global import Lldp_globalArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -63,7 +64,7 @@ class Lldp_globalFacts(object):
                     </protocols>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/lldp_interfaces/lldp_interfaces.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.lldp_interfaces.lldp_interfaces import Lldp_interfacesArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -65,7 +66,7 @@ class Lldp_interfacesFacts(object):
                     </protocols>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data, errors='surrogate_then_replace'))

lib/ansible/module_utils/network/junos/facts/vlans/vlans.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from ansible.module_utils._text import to_bytes
 from ansible.module_utils.network.common import utils
 from ansible.module_utils.network.junos.argspec.vlans.vlans import VlansArgs
+from ansible.module_utils.network.junos.utils.utils import get_resource_config
 from ansible.module_utils.six import string_types
 try:
     from lxml import etree
@@ -61,7 +62,7 @@ class VlansFacts(object):
                   </vlans>
                 </configuration>
                 """
-            data = connection.get_configuration(filter=config_filter)
+            data = get_resource_config(connection, config_filter=config_filter)
 
         if isinstance(data, string_types):
             data = etree.fromstring(to_bytes(data,

lib/ansible/module_utils/network/junos/utils/utils.py

@@ -5,3 +5,24 @@
 # (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 # utils
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.module_utils.network.junos.junos import tostring
+try:
+    from ncclient.xml_ import new_ele, to_ele, to_xml
+    HAS_NCCLIENT = True
+except ImportError:
+    HAS_NCCLIENT = False
+
+
+def get_resource_config(connection, config_filter=None, attrib=None):
+
+    if attrib is None:
+        attrib = {'inherit': 'inherit'}
+
+    get_ele = new_ele('get-configuration', attrib)
+    if config_filter:
+        get_ele.append(to_ele(config_filter))
+
+    return connection.execute_rpc(tostring(get_ele))

lib/ansible/modules/network/junos/junos_interfaces.py

@@ -316,7 +316,12 @@ def main():
 
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=InterfacesArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Interfaces(module).execute_module()

lib/ansible/modules/network/junos/junos_l2_interfaces.py

@@ -391,7 +391,12 @@ def main():
 
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=L2_interfacesArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = L2_interfaces(module).execute_module()

lib/ansible/modules/network/junos/junos_l3_interfaces.py

@@ -392,7 +392,12 @@ def main():
 
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=L3_interfacesArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = L3_interfaces(module).execute_module()

lib/ansible/modules/network/junos/junos_lacp.py

@@ -178,7 +178,11 @@ def main():
 
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',))]
+
     module = AnsibleModule(argument_spec=LacpArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Lacp(module).execute_module()

lib/ansible/modules/network/junos/junos_lacp_interfaces.py

@@ -504,7 +504,12 @@ def main():
     Main entry point for module execution
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=Lacp_interfacesArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Lacp_interfaces(module).execute_module()

lib/ansible/modules/network/junos/junos_lag_interfaces.py

@@ -333,7 +333,12 @@ def main():
 
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=Lag_interfacesArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Lag_interfaces(module).execute_module()

lib/ansible/modules/network/junos/junos_lldp_global.py

@@ -185,7 +185,11 @@ def main():
     Main entry point for module execution
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',))]
+
     module = AnsibleModule(argument_spec=Lldp_globalArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Lldp_global(module).execute_module()

lib/ansible/modules/network/junos/junos_lldp_interfaces.py

@@ -213,7 +213,12 @@ def main():
     Main entry point for module execution
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=Lldp_interfacesArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Lldp_interfaces(module).execute_module()

lib/ansible/modules/network/junos/junos_vlans.py

@@ -267,7 +267,12 @@ def main():
 
     :returns: the result form module invocation
     """
+    required_if = [('state', 'merged', ('config',)),
+                   ('state', 'replaced', ('config',)),
+                   ('state', 'overridden', ('config',))]
+
     module = AnsibleModule(argument_spec=VlansArgs.argument_spec,
+                           required_if=required_if,
                            supports_check_mode=True)
 
     result = Vlans(module).execute_module()

test/integration/targets/junos_interfaces/tests/netconf/groups.yaml

@@ -0,0 +1,49 @@
+---
+- debug:
+    msg: "START junos_interfaces groups integration tests on connection={{ ansible_connection }}"
+
+- include_tasks: _remove_config.yaml
+
+- set_fact:
+    expected_group_output:
+      - name: ge-0/0/11
+        description: "within test group"
+        enable: true
+      - name: ge-0/0/12
+        description: "global interface config"
+        enable: true
+      - name: fxp0
+        enable: true
+
+- name: "Teardown delete interface configuration"
+  junos_config: &delete_interface_config
+    lines:
+      - delete apply-groups test
+      - delete groups test interfaces ge-0/0/11
+      - delete interfaces ge-0/0/12
+
+- block:
+    - name: "Setup interface configuration"
+      junos_config:
+        lines:
+          - set groups test interfaces ge-0/0/11 description "within test group"
+          - set apply-groups test
+          - set interfaces ge-0/0/12 description "global interface config"
+
+    - name: "get junos interfaces facts"
+      junos_facts:
+        gather_subset: min
+        gather_network_resources: interfaces
+      register: result
+
+    - name: Assert the configuration is reflected on host
+      assert:
+        that:
+          - "{{ expected_group_output | symmetric_difference(result['ansible_facts']['ansible_network_resources']['interfaces'])|length == 0 }}"
+
+  always:
+    - name: "Teardown delete interface configuration"
+      junos_config: *delete_interface_config
+
+- debug:
+    msg: "END junos_interfaces merged integration tests on connection={{ ansible_connection }}"