Add netconf_get module (#39869)

* Add netconf_get module

Implements part-1 of proposal #104
https://github.com/ansible/proposals/issues/104

*  Add netconf_get module
*  Refactor `get`, `get_config`, `lock`, `unlock`
   and `discard_changes` netconf plugin api's
*  Add netconf module_utils file which netconf module
   related common functions
*  Refactor junos and iosxr netconf plugins

* Fix source option handling

* Fix review comments

* Update botmeta file

* Update review comments and add support for lock

* Lock update fix

* Fix CI issue

* Add integration test and minor fixes

* Fix review comments

* Fix CI failure

* Fix CI issues

* Fix CI issues

* Fix review comments and update integration test

* Fix review comments

* Fix review comments

* Fix review comments

Fix reveiw comments

.github/BOTMETA.yml

@@ -476,7 +476,8 @@ files:
   $modules/network/layer2/: $team_networking
   $modules/network/layer3/: $team_networking
   $modules/network/meraki/: $team_meraki
-  $modules/network/netconf/netconf_config.py: ganeshrn lpenz userlerueda
+  $modules/network/netconf/netconf_config.py: lpenz userlerueda $team_networking
+  $modules/network/netconf/netconf_get.py: wisotzky $team_networking
   $modules/network/netscaler/: $team_netscaler
   $modules/network/netvisor/: $team_netvisor
   $modules/network/nuage/: pdellaert
@@ -864,6 +865,9 @@ files:
   $module_utils/network/meraki:
     maintainers: $team_meraki
     labels: networking
+  $module_utils/network/netconf:
+    maintainers: $team_networking
+    labels: networking
   $module_utils/network/netscaler:
     maintainers: $team_netscaler
     labels: networking

lib/ansible/module_utils/network/common/netconf.py

@@ -30,6 +30,12 @@ import sys
 from ansible.module_utils._text import to_text, to_bytes
 from ansible.module_utils.connection import Connection, ConnectionError
 
+try:
+    from ncclient.xml_ import NCElement
+    HAS_NCCLIENT = True
+except ImportError:
+    HAS_NCCLIENT = False
+
 try:
     from lxml.etree import Element, fromstring, XMLSyntaxError
 except ImportError:
@@ -100,3 +106,40 @@ class NetconfConnection(Connection):
                 return warnings
             except XMLSyntaxError:
                 raise ConnectionError(rpc_error)
+
+
+def transform_reply():
+    reply = '''<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+    <xsl:output method="xml" indent="no"/>
+
+    <xsl:template match="/|comment()|processing-instruction()">
+        <xsl:copy>
+            <xsl:apply-templates/>
+        </xsl:copy>
+    </xsl:template>
+
+    <xsl:template match="*">
+        <xsl:element name="{local-name()}">
+            <xsl:apply-templates select="@*|node()"/>
+        </xsl:element>
+    </xsl:template>
+
+    <xsl:template match="@*">
+        <xsl:attribute name="{local-name()}">
+            <xsl:value-of select="."/>
+        </xsl:attribute>
+    </xsl:template>
+    </xsl:stylesheet>
+    '''
+    if sys.version < '3':
+        return reply
+    else:
+        return reply.encode('UTF-8')
+
+
+# Note: Workaround for ncclient 0.5.3
+def remove_namespaces(data):
+    if not HAS_NCCLIENT:
+        raise ImportError("ncclient is required but does not appear to be installed.  "
+                          "It can be installed using `pip install ncclient`")
+    return NCElement(data, transform_reply()).data_xml

lib/ansible/module_utils/network/netconf/netconf.py

@@ -0,0 +1,106 @@
+#
+# (c) 2018 Red Hat, Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+import json
+
+from contextlib import contextmanager
+
+from ansible.module_utils._text import to_text
+from ansible.module_utils.connection import Connection, ConnectionError
+from ansible.module_utils.network.common.netconf import NetconfConnection
+
+
+def get_connection(module):
+    if hasattr(module, '_netconf_connection'):
+        return module._netconf_connection
+
+    capabilities = get_capabilities(module)
+    network_api = capabilities.get('network_api')
+    if network_api == 'netconf':
+        module._netconf_connection = NetconfConnection(module._socket_path)
+    else:
+        module.fail_json(msg='Invalid connection type %s' % network_api)
+
+    return module._netconf_connection
+
+
+def get_capabilities(module):
+    if hasattr(module, '_netconf_capabilities'):
+        return module._netconf_capabilities
+
+    capabilities = Connection(module._socket_path).get_capabilities()
+    module._netconf_capabilities = json.loads(capabilities)
+    return module._netconf_capabilities
+
+
+def lock_configuration(x, target=None):
+    conn = get_connection(x)
+    return conn.lock(target=target)
+
+
+def unlock_configuration(x, target=None):
+    conn = get_connection(x)
+    return conn.unlock(target=target)
+
+
+@contextmanager
+def locked_config(module, target=None):
+    try:
+        lock_configuration(module, target=target)
+        yield
+    finally:
+        unlock_configuration(module, target=target)
+
+
+def get_config(module, source, filter, lock=False):
+    conn = get_connection(module)
+    try:
+        locked = False
+        if lock:
+            conn.lock(target='running')
+            locked = True
+        response = conn.get_config(source=source, filter=filter)
+
+    except ConnectionError as e:
+        module.fail_json(msg=to_text(e, errors='surrogate_then_replace').strip())
+
+    finally:
+        if locked:
+            conn.unlock(target='running')
+
+    return response
+
+
+def get(module, filter, lock=False):
+    conn = get_connection(module)
+    try:
+        locked = False
+        if lock:
+            conn.lock(target='running')
+            locked = True
+
+        response = conn.get(filter=filter)
+
+    except ConnectionError as e:
+        module.fail_json(msg=to_text(e, errors='surrogate_then_replace').strip())
+
+    finally:
+        if locked:
+            conn.unlock(target='running')
+
+    return response

lib/ansible/modules/network/netconf/netconf_get.py

@@ -0,0 +1,262 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2018, Ansible by Red Hat, inc
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'network'}
+
+
+DOCUMENTATION = """
+---
+module: netconf_get
+version_added: "2.6"
+author:
+    - "Ganesh Nalawade (@ganeshrn)"
+    - "Sven Wisotzky (@wisotzky)"
+short_description: Fetch configuration/state data from NETCONF enabled network devices.
+description:
+    - NETCONF is a network management protocol developed and standardized by
+      the IETF. It is documented in RFC 6241.
+    - This module allows the user to fetch configuration and state data from NETCONF
+      enabled network devices.
+options:
+  source:
+    description:
+      - This argument specifies the datastore from which configuration data should be fetched.
+        Valid values are I(running), I(candidate) and I(startup). If the C(source) value is not
+        set both configuration and state information are returned in response from running datastore.
+    choices: ['running', 'candidate', 'startup']
+  filter:
+    description:
+      - This argument specifies the XML string which acts as a filter to restrict the portions of
+        the data to be are retrieved from the remote device. If this option is not specified entire
+        configuration or state data is returned in result depending on the value of C(source)
+        option. The C(filter) value can be either XML string or XPath, if the filter is in
+        XPath format the NETCONF server running on remote host should support xpath capability
+        else it will result in an error.
+  display:
+    description:
+      - Encoding scheme to use when serializing output from the device. The option I(json) will
+        serialize the output as JSON data. If the option value is I(json) it requires jxmlease
+        to be installed on control node. The option I(pretty) is similar to received XML response
+        but is using human readable format (spaces, new lines). The option value I(xml) is similar
+        to received XML response but removes all XML namespaces.
+    choices: ['json', 'pretty', 'xml']
+  lock:
+    description:
+      - Instructs the module to explicitly lock the datastore specified as C(source) before fetching
+        configuration and/or state information from remote host. If the value is I(never) in that case
+        the C(source) datastore is never locked, if the value is I(if-supported) the C(source) datastore
+        is locked only if the Netconf server running on remote host supports locking of that datastore,
+        if the lock on C(source) datastore is not supported module will report appropriate error before
+        executing lock. If the value is I(always) the lock operation on C(source) datastore will always
+        be executed irrespective if the remote host supports it or not, if it doesn't the module with
+        fail will the execption message received from remote host and might vary based on the platform.
+    default: 'never'
+    choices: ['never', 'always', 'if-supported']
+requirements:
+  - ncclient (>=v0.5.2)
+  - jxmlease
+
+notes:
+  - This module requires the NETCONF system service be enabled on
+    the remote device being managed.
+  - This module supports the use of connection=netconf
+"""
+
+EXAMPLES = """
+- name: Get running configuration and state data
+  netconf_get:
+
+- name: Get configuration and state data from startup datastore
+  netconf_get:
+    source: startup
+
+- name: Get system configuration data from running datastore state (junos)
+  netconf_get:
+    source: running
+    filter: <configuration><system></system></configuration>
+
+- name: Get configuration and state data in JSON format
+  netconf_get:
+    display: json
+
+- name: get schema list using subtree w/ namespaces
+  netconf_get:
+    format: json
+    filter: <netconf-state xmlns="urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring"><schemas><schema/></schemas></netconf-state>
+    lock: False
+
+- name: get schema list using xpath
+  netconf_get:
+    format: json
+    filter: /netconf-state/schemas/schema
+
+- name: get interface confiugration with filter (iosxr)
+  netconf_get:
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+
+- name: Get system configuration data from running datastore state (sros)
+  netconf_get:
+    source: running
+    filter: <state xmlns="urn:nokia.com:sros:ns:yang:sr:conf"/>
+    lock: True
+
+- name: Get state data (sros)
+  netconf_get:
+    filter: <state xmlns="urn:nokia.com:sros:ns:yang:sr:state"/>
+"""
+
+RETURN = """
+stdout:
+  description: The raw XML string containing configuration or state data
+               received from the underlying ncclient library.
+  returned: always apart from low-level errors (such as action plugin)
+  type: string
+  sample: '...'
+stdout_lines:
+  description: The value of stdout split into a list
+  returned: always apart from low-level errors (such as action plugin)
+  type: list
+  sample: ['...', '...']
+output:
+  description: Based on the value of display option will return either the set of
+               transformed XML to JSON format from the RPC response with type dict
+               or pretty XML string response (human-readable) or response with
+               namespace removed from XML string.
+  returned: when the display format is selected as JSON it is returned as dict type, if the
+            display format is xml or pretty pretty it is retured as a string apart from low-level
+            errors (such as action plugin).
+  type: complex
+  contains:
+    formatted_output:
+      - Contains formatted response received from remote host as per the value in display format.
+"""
+import sys
+
+try:
+    from lxml.etree import Element, SubElement, tostring, fromstring, XMLSyntaxError
+except ImportError:
+    from xml.etree.ElementTree import Element, SubElement, tostring, fromstring
+    if sys.version_info < (2, 7):
+        from xml.parsers.expat import ExpatError as XMLSyntaxError
+    else:
+        from xml.etree.ElementTree import ParseError as XMLSyntaxError
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.network.netconf.netconf import get_capabilities, locked_config, get_config, get
+from ansible.module_utils.network.common.netconf import remove_namespaces
+
+try:
+    import jxmlease
+    HAS_JXMLEASE = True
+except ImportError:
+    HAS_JXMLEASE = False
+
+
+def get_filter_type(filter):
+    if not filter:
+        return None
+    else:
+        try:
+            fromstring(filter)
+            return 'subtree'
+        except XMLSyntaxError:
+            return 'xpath'
+
+
+def main():
+    """entry point for module execution
+    """
+    argument_spec = dict(
+        source=dict(choices=['running', 'candidate', 'startup']),
+        filter=dict(),
+        display=dict(choices=['json', 'pretty', 'xml']),
+        lock=dict(default='never', choices=['never', 'always', 'if-supported'])
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec,
+                           supports_check_mode=True)
+
+    capabilities = get_capabilities(module)
+    operations = capabilities['device_operations']
+
+    source = module.params['source']
+    filter = module.params['filter']
+    filter_type = get_filter_type(filter)
+    lock = module.params['lock']
+    display = module.params['display']
+
+    if source == 'candidate' and not operations.get('supports_commit', False):
+        module.fail_json(msg='candidate source is not supported on this device')
+
+    if source == 'startup' and not operations.get('supports_startup', False):
+        module.fail_json(msg='startup source is not supported on this device')
+
+    if filter_type == 'xpath' and not operations.get('supports_xpath', False):
+        module.fail_json(msg="filter value '%s' of type xpath is not supported on this device" % filter)
+
+    execute_lock = True if lock in ('always', 'if-supported') else False
+
+    if lock == 'always' and not operations.get('supports_lock', False):
+        module.fail_json(msg='lock operation is not supported on this device')
+
+    if execute_lock:
+        if source is None:
+            # if source is None, in that case operation is 'get' and `get` supports
+            # fetching data only from running datastore
+            if 'running' not in operations.get('lock_datastore', []):
+                # lock is not supported, don't execute lock operation
+                if lock == 'if-supported':
+                    execute_lock = False
+                else:
+                    module.warn("lock operation on 'running' source is not supported on this device")
+        else:
+            if source not in operations.get('lock_datastore', []):
+                if lock == 'if-supported':
+                    # lock is not supported, don't execute lock operation
+                    execute_lock = False
+                else:
+                    module.warn("lock operation on '%s' source is not supported on this device" % source)
+
+    if display == 'json' and not HAS_JXMLEASE:
+        module.fail_json(msg='jxmlease is required to display response in json format'
+                             'but does not appear to be installed. '
+                             'It can be installed using `pip install jxmlease`')
+
+    filter_spec = (filter_type, filter) if filter_type else None
+
+    if source is not None:
+        response = get_config(module, source, filter_spec, execute_lock)
+    else:
+        response = get(module, filter_spec, execute_lock)
+
+    xml_resp = tostring(response)
+    output = None
+
+    if display == 'xml':
+        output = remove_namespaces(xml_resp)
+    elif display == 'json':
+        try:
+            output = jxmlease.parse(xml_resp)
+        except:
+            raise ValueError(xml_resp)
+    elif display == 'pretty':
+        output = tostring(response, pretty_print=True)
+
+    result = {
+        'stdout': xml_resp,
+        'output': output
+    }
+
+    module.exit_json(**result)
+
+if __name__ == '__main__':
+    main()

lib/ansible/plugins/netconf/__init__.py

@@ -110,22 +110,35 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
             raise Exception(to_xml(msg))
 
     @ensure_connected
-    def get_config(self, *args, **kwargs):
+    def get_config(self, source=None, filter=None):
-        """Retrieve all or part of a specified configuration.
+        """Retrieve all or part of a specified configuration
-           :source: name of the configuration datastore being queried
+           (by default entire configuration is retrieved).
-           :filter: specifies the portion of the configuration to retrieve
+
-           (by default entire configuration is retrieved)"""
+        :param source: Name of the configuration datastore being queried, defaults to running datastore
-        resp = self.m.get_config(*args, **kwargs)
+        :param filter: This argument specifies the portion of the configuration data to retrieve
+        :return: Returns xml string containing the RPC response received from remote host
+        """
+        if isinstance(filter, list):
+            filter = tuple(filter)
+
+        if not source:
+            source = 'running'
+        resp = self.m.get_config(source=source, filter=filter)
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @ensure_connected
-    def get(self, *args, **kwargs):
+    def get(self, filter=None):
-        """Retrieve running configuration and device state information.
+        """Retrieve device configuration and state information.
-        *filter* specifies the portion of the configuration to retrieve
+
-        (by default entire configuration is retrieved)
+        :param filter: This argument specifies the portion of the state data to retrieve
+                        (by default entire state data is retrieved)
+        :return: Returns xml string containing the RPC response received from remote host
         """
-        resp = self.m.get(*args, **kwargs)
+        if isinstance(filter, list):
-        return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
+            filter = tuple(filter)
+        resp = self.m.get(filter=filter)
+        response = resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
+        return response
 
     @ensure_connected
     def edit_config(self, *args, **kwargs):
@@ -162,26 +175,42 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @ensure_connected
-    def lock(self, *args, **kwargs):
+    def lock(self, target=None):
-        """Allows the client to lock the configuration system of a device.
-        *target* is the name of the configuration datastore to lock
         """
-        resp = self.m.lock(*args, **kwargs)
+        Allows the client to lock the configuration system of a device.
+        :param target: is the name of the configuration datastore to lock,
+                        defaults to candidate datastore
+        :return: Returns xml string containing the RPC response received from remote host
+        """
+        if not target:
+            target = 'candidate'
+        resp = self.m.lock(target=target)
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @ensure_connected
-    def unlock(self, *args, **kwargs):
+    def unlock(self, target=None):
+        """
+        Release a configuration lock, previously obtained with the lock operation.
+        :param target: is the name of the configuration datastore to unlock,
+                       defaults to candidate datastore
+        :return: Returns xml string containing the RPC response received from remote host
+        """
         """Release a configuration lock, previously obtained with the lock operation.
         :target: is the name of the configuration datastore to unlock
         """
-        resp = self.m.unlock(*args, **kwargs)
+        if not target:
+            target = 'candidate'
+        resp = self.m.unlock(target=target)
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @ensure_connected
-    def discard_changes(self, *args, **kwargs):
+    def discard_changes(self):
-        """Revert the candidate configuration to the currently running configuration.
+        """
-        Any uncommitted changes are discarded."""
+        Revert the candidate configuration to the currently running configuration.
-        resp = self.m.discard_changes(*args, **kwargs)
+        Any uncommitted changes are discarded.
+        :return: Returns xml string containing the RPC response received from remote host
+        """
+        resp = self.m.discard_changes()
         return resp.data_xml if hasattr(resp, 'data_xml') else resp.xml
 
     @ensure_connected
@@ -245,4 +274,28 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         """Fetch file over scp from remote device"""
         pass
 
+    def get_device_operations(self, server_capabilities):
+        operations = {}
+        capabilities = '\n'.join(server_capabilities)
+        operations['supports_commit'] = True if ':candidate' in capabilities else False
+        operations['supports_defaults'] = True if ':with-defaults' in capabilities else False
+        operations['supports_confirm_commit'] = True if ':confirmed-commit' in capabilities else False
+        operations['supports_startup'] = True if ':startup' in capabilities else False
+        operations['supports_xpath'] = True if ':xpath' in capabilities else False
+        operations['supports_writeable_running'] = True if ':writable-running' in capabilities else False
+
+        operations['lock_datastore'] = []
+        if operations['supports_writeable_running']:
+            operations['lock_datastore'].append('running')
+
+        if operations['supports_commit']:
+            operations['lock_datastore'].append('candidate')
+
+        if operations['supports_startup']:
+            operations['lock_datastore'].append('startup')
+
+        operations['supports_lock'] = True if len(operations['lock_datastore']) else False
+
+        return operations
+
 # TODO Restore .xml, when ncclient supports it for all platforms

lib/ansible/plugins/netconf/default.py

@@ -48,4 +48,5 @@ class Netconf(NetconfBase):
         result['server_capabilities'] = [c for c in self.m.server_capabilities]
         result['client_capabilities'] = [c for c in self.m.client_capabilities]
         result['session_id'] = self.m.session_id
+        result['device_operations'] = self.get_device_operations(result['server_capabilities'])
         return json.dumps(result)

lib/ansible/plugins/netconf/iosxr.py

@@ -22,12 +22,10 @@ __metaclass__ = type
 
 import json
 import re
-import sys
 import collections
-from io import BytesIO
-from ansible.module_utils.six import StringIO
 
 from ansible import constants as C
+from ansible.module_utils.network.common.netconf import remove_namespaces
 from ansible.module_utils.network.iosxr.iosxr import build_xml
 from ansible.errors import AnsibleConnectionFailure, AnsibleError
 from ansible.plugins.netconf import NetconfBase
@@ -47,45 +45,6 @@ except ImportError:
     raise AnsibleError("lxml is not installed")
 
 
-def transform_reply():
-    reply = '''<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
-    <xsl:output method="xml" indent="no"/>
-
-    <xsl:template match="/|comment()|processing-instruction()">
-        <xsl:copy>
-            <xsl:apply-templates/>
-        </xsl:copy>
-    </xsl:template>
-
-    <xsl:template match="*">
-        <xsl:element name="{local-name()}">
-            <xsl:apply-templates select="@*|node()"/>
-        </xsl:element>
-    </xsl:template>
-
-    <xsl:template match="@*">
-        <xsl:attribute name="{local-name()}">
-            <xsl:value-of select="."/>
-        </xsl:attribute>
-    </xsl:template>
-    </xsl:stylesheet>
-    '''
-    if sys.version < '3':
-        return reply
-    else:
-        return reply.encode('UTF-8')
-
-
-# Note: Workaround for ncclient 0.5.3
-def remove_namespaces(rpc_reply):
-    xslt = transform_reply()
-    parser = etree.XMLParser(remove_blank_text=True)
-    xslt_doc = etree.parse(BytesIO(xslt), parser)
-    transform = etree.XSLT(xslt_doc)
-
-    return etree.fromstring(str(transform(etree.parse(StringIO(str(rpc_reply))))))
-
-
 class Netconf(NetconfBase):
 
     @ensure_connected
@@ -129,7 +88,7 @@ class Netconf(NetconfBase):
         result['server_capabilities'] = [c for c in self.m.server_capabilities]
         result['client_capabilities'] = [c for c in self.m.client_capabilities]
         result['session_id'] = self.m.session_id
-
+        result['device_operations'] = self.get_device_operations(result['server_capabilities'])
         return json.dumps(result)
 
     @staticmethod
@@ -161,18 +120,22 @@ class Netconf(NetconfBase):
 
     # TODO: change .xml to .data_xml, when ncclient supports data_xml on all platforms
     @ensure_connected
-    def get(self, *args, **kwargs):
+    def get(self, filter=None):
+        if isinstance(filter, list):
+            filter = tuple(filter)
         try:
-            response = self.m.get(*args, **kwargs)
+            response = self.m.get(filter=filter)
-            return to_xml(remove_namespaces(response))
+            return remove_namespaces(response)
         except RPCError as exc:
             raise Exception(to_xml(exc.xml))
 
     @ensure_connected
-    def get_config(self, *args, **kwargs):
+    def get_config(self, source=None, filter=None):
+        if isinstance(filter, list):
+            filter = tuple(filter)
         try:
-            response = self.m.get_config(*args, **kwargs)
+            response = self.m.get_config(source=source, filter=filter)
-            return to_xml(remove_namespaces(response))
+            return remove_namespaces(response)
         except RPCError as exc:
             raise Exception(to_xml(exc.xml))
 
@@ -180,7 +143,7 @@ class Netconf(NetconfBase):
     def edit_config(self, *args, **kwargs):
         try:
             response = self.m.edit_config(*args, **kwargs)
-            return to_xml(remove_namespaces(response))
+            return remove_namespaces(response)
         except RPCError as exc:
             raise Exception(to_xml(exc.xml))
 
@@ -188,7 +151,7 @@ class Netconf(NetconfBase):
     def commit(self, *args, **kwargs):
         try:
             response = self.m.commit(*args, **kwargs)
-            return to_xml(remove_namespaces(response))
+            return remove_namespaces(response)
         except RPCError as exc:
             raise Exception(to_xml(exc.xml))
 
@@ -196,14 +159,14 @@ class Netconf(NetconfBase):
     def validate(self, *args, **kwargs):
         try:
             response = self.m.validate(*args, **kwargs)
-            return to_xml(remove_namespaces(response))
+            return remove_namespaces(response)
         except RPCError as exc:
             raise Exception(to_xml(exc.xml))
 
     @ensure_connected
-    def discard_changes(self, *args, **kwargs):
+    def discard_changes(self):
         try:
-            response = self.m.discard_changes(*args, **kwargs)
+            response = self.m.discard_changes()
-            return to_xml(remove_namespaces(response))
+            return remove_namespaces(response)
         except RPCError as exc:
             raise Exception(to_xml(exc.xml))

lib/ansible/plugins/netconf/junos.py

@@ -92,6 +92,7 @@ class Netconf(NetconfBase):
         result['server_capabilities'] = [c for c in self.m.server_capabilities]
         result['client_capabilities'] = [c for c in self.m.client_capabilities]
         result['session_id'] = self.m.session_id
+        result['device_operations'] = self.get_device_operations(result['server_capabilities'])
         return json.dumps(result)
 
     @staticmethod
@@ -143,44 +144,3 @@ class Netconf(NetconfBase):
     def reboot(self):
         """reboot the device"""
         return self.m.reboot().data_xml
-
-    @ensure_connected
-    def halt(self):
-        """reboot the device"""
-        return self.m.halt().data_xml
-
-    @ensure_connected
-    def get(self, *args, **kwargs):
-        try:
-            return self.m.get(*args, **kwargs).data_xml
-        except RPCError as exc:
-            raise Exception(to_xml(exc.xml))
-
-    @ensure_connected
-    def get_config(self, *args, **kwargs):
-        try:
-            return self.m.get_config(*args, **kwargs).data_xml
-        except RPCError as exc:
-            raise Exception(to_xml(exc.xml))
-
-    @ensure_connected
-    def edit_config(self, *args, **kwargs):
-        try:
-            self.m.edit_config(*args, **kwargs).data_xml
-        except RPCError as exc:
-            raise Exception(to_xml(exc.xml))
-
-    @ensure_connected
-    def commit(self, *args, **kwargs):
-        try:
-            return self.m.commit(*args, **kwargs).data_xml
-        except RPCError as exc:
-            raise Exception(to_xml(exc.xml))
-
-    @ensure_connected
-    def validate(self, *args, **kwargs):
-        return self.m.validate(*args, **kwargs).data_xml
-
-    @ensure_connected
-    def discard_changes(self, *args, **kwargs):
-        return self.m.discard_changes(*args, **kwargs).data_xml

test/integration/targets/netconf_get/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+testcase: "*"

test/integration/targets/netconf_get/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - { role: prepare_junos_tests, when: ansible_network_os == 'junos' }
+  - { role: prepare_iosxr_tests, when: ansible_network_os == 'iosxr' }

test/integration/targets/netconf_get/tasks/iosxr.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/iosxr"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }}"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_get/tasks/junos.yaml

@@ -0,0 +1,16 @@
+---
+- name: collect all netconf test cases
+  find:
+    paths: "{{ role_path }}/tests/junos"
+    patterns: "{{ testcase }}.yaml"
+  register: test_cases
+  connection: local
+
+- name: set test_items
+  set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}"
+
+- name: run test case (connection=netconf)
+  include: "{{ test_case_to_run }} ansible_connection=netconf"
+  with_items: "{{ test_items }}"
+  loop_control:
+    loop_var: test_case_to_run

test/integration/targets/netconf_get/tasks/main.yaml

@@ -0,0 +1,3 @@
+---
+- { include: junos.yaml, when: ansible_network_os == 'junos', tags: ['netconf'] }
+- { include: iosxr.yaml, when: ansible_network_os == 'iosxr', tags: ['netconf'] }

test/integration/targets/netconf_get/tests/iosxr/basic.yaml

@@ -0,0 +1,163 @@
+---
+- debug: msg="START netconf_get iosxr/basic.yaml on connection={{ ansible_connection }}"
+
+- name: setup interface
+  iosxr_config:
+    commands:
+      - description this is test interface Loopback999
+      - no shutdown
+    parents:
+      - interface Loopback999
+    match: none
+  connection: network_cli
+
+- name: get running interface confiugration with filter
+  netconf_get:
+    source: running
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+      - "'<usernames>' not in result.stdout"
+
+- name: test lock=never, get-config, running interface confiugration with filter without lock
+  netconf_get:
+    source: running
+    lock: never
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+      - "'<usernames>' not in result.stdout"
+
+- name: test lock=if-supported, get-config, running interface confiugration with filter without lock
+  netconf_get:
+    source: running
+    lock: if-supported
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+      - "'<usernames>' not in result.stdout"
+
+- name: Failure scenario, get-config information with lock
+  netconf_get:
+    source: running
+    lock: always
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'<bad-element>running</bad-element>' in result.msg"
+
+- name: Failure scenario, fetch config from startup
+  netconf_get:
+    source: startup
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'startup source is not supported' in result.msg"
+
+- name: test get, information from running datastore without lock
+  netconf_get:
+    lock: never
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+
+- name: test get, information from running datastore with lock if supported
+  netconf_get:
+    lock: if-supported
+    filter: <interface-configurations xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-ifmgr-cfg"></interface-configurations>
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "'<description>this is test interface Loopback999</description>' in result.stdout"
+
+- name: Failure scenario, get information from running with lock
+  netconf_get:
+    lock: always
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'<bad-element>running</bad-element>' in result.msg"
+
+- name: get configuration and state data in json format
+  netconf_get:
+    source: running
+    display: json
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output']['rpc-reply']['data']['aaa'] is defined}}"
+
+- name: get configuration data in xml pretty format
+  netconf_get:
+    source: running
+    display: pretty
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+
+- name: get configuration data in xml with namespace stripped
+  netconf_get:
+    source: running
+    display: xml
+  register: result
+  connection: netconf
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+      - "{{ 'xmlns' not in result.output }}"
+
+- name: Failure scenario, unsupported filter
+  netconf_get:
+    filter: configuration/state
+  register: result
+  ignore_errors: True
+  connection: netconf
+
+- assert:
+    that:
+      - "'filter value \\'configuration/state\\' of type xpath is not supported' in result.msg"
+
+- name: setup - teardown
+  iosxr_config:
+    commands:
+      - no description
+      - shutdown
+    parents:
+      - interface Loopback999
+    match: none
+  connection: network_cli
+
+- debug: msg="END netconf_get iosxr/basic.yaml on connection={{ ansible_connection }}"

test/integration/targets/netconf_get/tests/junos/basic.yaml

@@ -0,0 +1,126 @@
+---
+- debug: msg="START netconf_get junos/basic.yaml on connection={{ ansible_connection }}"
+
+- name: Configure syslog file - setup
+  junos_config:
+    lines:
+    - set system syslog file test1 any any
+  register: result
+
+- name: Get system configuration data from running datastore state
+  netconf_get:
+    source: running
+    filter: <configuration><system><syslog></syslog></system></configuration>
+  register: result
+
+- assert:
+    that:
+      - "'<name>test1</name>' in result.stdout"
+      - "'<name>any</name>' in result.stdout"
+      - "'<any/>' in result.stdout"
+      - "'<login>' not in result.stdout"
+      - "'<interface>' not in result.stdout"
+
+- name: Failure scenario, fetch config from startup
+  netconf_get:
+    source: startup
+  register: result
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'startup source is not supported' in result.msg"
+
+- name: Failure scenario, fetch config from running with lock
+  netconf_get:
+    lock: always
+    source: running
+  register: result
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'syntax error' in result.msg"
+
+- name: Get system configuration data from running datastore state and lock if-supported
+  netconf_get:
+    source: running
+    filter: <configuration><system><syslog></syslog></system></configuration>
+    lock: if-supported
+  register: result
+
+- assert:
+    that:
+      - "'<name>test1</name>' in result.stdout"
+      - "'<name>any</name>' in result.stdout"
+      - "'<any/>' in result.stdout"
+      - "'<login>' not in result.stdout"
+      - "'<interface>' not in result.stdout"
+
+- name: get configuration and state data in json format
+  netconf_get:
+    source: running
+    display: json
+  register: result
+
+- assert:
+    that:
+      - "{{ result['output']['rpc-reply']['data']['configuration'] is defined}}"
+
+- name: get configuration and state data in xml pretty format
+  netconf_get:
+    source: running
+    display: pretty
+  register: result
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+
+- name: get configuration data in xml with namespace stripped
+  netconf_get:
+    source: running
+    display: xml
+  register: result
+
+- assert:
+    that:
+      - "{{ result['output'] is defined}}"
+      - "{{ 'xmlns' not in result.output }}"
+
+- name: get configuration and state data without datastore lock
+  netconf_get:
+    lock: never
+  register: result
+
+- assert:
+    that:
+      - "'<database-status-information>' in result.stdout"
+      - "'</configuration>' in result.stdout"
+
+- name: get configuration and state data and lock data-store if supported
+  netconf_get:
+    lock: if-supported
+  register: result
+
+- assert:
+    that:
+      - "'<database-status-information>' in result.stdout"
+      - "'</configuration>' in result.stdout"
+
+- name: Failure scenario, unsupported filter
+  netconf_get:
+    filter: configuration/state
+  register: result
+  ignore_errors: True
+
+- assert:
+    that:
+      - "'filter value \\'configuration/state\\' of type xpath is not supported' in result.msg"
+
+- name: Configure syslog file - teardown
+  junos_config:
+    lines:
+    - delete system syslog file test1 any any
+
+- debug: msg="END netconf_get junos/basic.yaml on connection={{ ansible_connection }}"