Netconf bytes fixes (#41607)

* Solve some bytes issues on iosxr

* Solve some bytes issues in junos

* Do the correct thing with tostring based on lxml or not

lib/ansible/module_utils/network/common/netconf.py

@@ -109,7 +109,7 @@ class NetconfConnection(Connection):
 
 
 def transform_reply():
-    reply = '''<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+    return b'''<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
     <xsl:output method="xml" indent="no"/>
 
     <xsl:template match="/|comment()|processing-instruction()">
@@ -131,10 +131,6 @@ def transform_reply():
     </xsl:template>
     </xsl:stylesheet>
     '''
-    if sys.version < '3':
-        return reply
-    else:
-        return reply.encode('UTF-8')
 
 
 # Note: Workaround for ncclient 0.5.3

lib/ansible/module_utils/network/iosxr/iosxr.py

@@ -260,31 +260,25 @@ def build_xml(container, xmap=None, params=None, opcode=None):
             for item in subtree_list:
                 container_ele.append(item)
 
-    return etree.tostring(root)
+    return etree.tostring(root, encoding='unicode')
 
 
 def etree_find(root, node):
     try:
-        element = etree.fromstring(root).find('.//' + to_bytes(node, errors='surrogate_then_replace').strip())
-    except Exception:
-        element = etree.fromstring(etree.tostring(root)).find('.//' + to_bytes(node, errors='surrogate_then_replace').strip())
+        root = etree.fromstring(to_bytes(root))
+    except (ValueError, etree.XMLSyntaxError):
+        pass
 
-    if element is not None:
-        return element
-
-    return None
+    return root.find('.//%s' % node.strip())
 
 
 def etree_findall(root, node):
     try:
-        element = etree.fromstring(root).findall('.//' + to_bytes(node, errors='surrogate_then_replace').strip())
-    except Exception:
-        element = etree.fromstring(etree.tostring(root)).findall('.//' + to_bytes(node, errors='surrogate_then_replace').strip())
+        root = etree.fromstring(to_bytes(root))
+    except (ValueError, etree.XMLSyntaxError):
+        pass
 
-    if element is not None:
-        return element
-
-    return None
+    return root.findall('.//%s' % node.strip())
 
 
 def is_cliconf(module):

lib/ansible/module_utils/network/junos/junos.py

@@ -27,10 +27,10 @@ from ansible.module_utils.network.common.netconf import NetconfConnection
 from ansible.module_utils._text import to_text
 
 try:
-    from lxml.etree import Element, SubElement, fromstring, tostring
+    from lxml.etree import Element, SubElement, tostring as xml_to_string
     HAS_LXML = True
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, fromstring, tostring
+    from xml.etree.ElementTree import Element, SubElement, tostring as xml_to_string
     HAS_LXML = False
 
 ACTIONS = frozenset(['merge', 'override', 'replace', 'update', 'set'])
@@ -62,6 +62,13 @@ junos_top_spec = {
 junos_argument_spec.update(junos_top_spec)
 
 
+def tostring(element, encoding='UTF-8'):
+    if HAS_LXML:
+        return xml_to_string(element, encoding='unicode')
+    else:
+        return to_text(xml_to_string(element, encoding), encoding=encoding)
+
+
 def get_provider_argspec():
     return junos_provider_spec
 

lib/ansible/modules/network/junos/junos_banner.py

@@ -104,15 +104,10 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_command.py

@@ -165,15 +165,15 @@ import shlex
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_text
 from ansible.module_utils.network.common.netconf import exec_rpc
-from ansible.module_utils.network.junos.junos import junos_argument_spec, get_configuration, get_connection, get_capabilities
+from ansible.module_utils.network.junos.junos import junos_argument_spec, get_configuration, get_connection, get_capabilities, tostring
 from ansible.module_utils.network.common.parsing import Conditional, FailedConditionalError
 from ansible.module_utils.six import string_types, iteritems
 
 
 try:
-    from lxml.etree import Element, SubElement, tostring
+    from lxml.etree import Element, SubElement
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, tostring
+    from xml.etree.ElementTree import Element, SubElement
 
 try:
     import jxmlease
@@ -233,7 +233,7 @@ def rpc(module, items):
         if fetch_config:
             reply = get_configuration(module, format=xattrs['format'])
         else:
-            reply = exec_rpc(module, to_text(tostring(element), errors='surrogate_then_replace'), ignore_warning=False)
+            reply = exec_rpc(module, tostring(element), ignore_warning=False)
 
         if xattrs['format'] == 'text':
             if fetch_config:

lib/ansible/modules/network/junos/junos_config.py

@@ -182,7 +182,7 @@ from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.netconf import exec_rpc
 from ansible.module_utils.network.junos.junos import get_diff, load_config, get_configuration
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
-from ansible.module_utils.network.junos.junos import junos_argument_spec, load_configuration, get_connection, tostring
+from ansible.module_utils.network.junos.junos import junos_argument_spec, load_configuration, tostring
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native, to_text
 

lib/ansible/modules/network/junos/junos_facts.py

@@ -81,16 +81,16 @@ ansible_facts:
 """
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.netconf import exec_rpc
-from ansible.module_utils.network.junos.junos import junos_argument_spec, get_param
+from ansible.module_utils.network.junos.junos import junos_argument_spec, get_param, tostring
 from ansible.module_utils.network.junos.junos import get_configuration, get_connection
 from ansible.module_utils._text import to_native
 from ansible.module_utils.six import iteritems
 
 
 try:
-    from lxml.etree import Element, SubElement, tostring
+    from lxml.etree import Element, SubElement
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, tostring
+    from xml.etree.ElementTree import Element, SubElement
 
 try:
     from jnpr.junos import Device

lib/ansible/modules/network/junos/junos_interface.py

@@ -195,14 +195,14 @@ from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.netconf import exec_rpc
 from ansible.module_utils.network.common.utils import remove_default_spec
 from ansible.module_utils.network.common.utils import conditional
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config, to_param_list
 
 try:
-    from lxml.etree import Element, SubElement, tostring
+    from lxml.etree import Element, SubElement
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, tostring
+    from xml.etree.ElementTree import Element, SubElement
 
 USE_PERSISTENT_CONNECTION = True
 

lib/ansible/modules/network/junos/junos_l2_interface.py

@@ -139,15 +139,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config, to_param_list
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_l3_interface.py

@@ -106,15 +106,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config, to_param_list
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_linkagg.py

@@ -164,15 +164,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config, get_configuration
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_lldp.py

@@ -101,15 +101,10 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_lldp_interface.py

@@ -97,14 +97,9 @@ import collections
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.junos.junos import junos_argument_spec
-from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
+from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele, tostring
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_logging.py

@@ -142,15 +142,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_rpc.py

@@ -95,15 +95,15 @@ output_lines:
 """
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.netconf import exec_rpc
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.six import iteritems
 
 USE_PERSISTENT_CONNECTION = True
 
 try:
-    from lxml.etree import Element, SubElement, tostring
+    from lxml.etree import Element, SubElement
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, tostring
+    from xml.etree.ElementTree import Element, SubElement
 
 
 def main():
@@ -153,7 +153,7 @@ def main():
 
     reply = exec_rpc(module, tostring(element), ignore_warning=False)
 
-    result['xml'] = str(tostring(reply))
+    result['xml'] = tostring(reply)
 
     if module.params['output'] == 'text':
         data = reply.find('.//output')
@@ -164,7 +164,7 @@ def main():
         result['output'] = module.from_json(reply.text.strip())
 
     else:
-        result['output'] = str(tostring(reply)).split('\n')
+        result['output'] = tostring(reply).split('\n')
 
     module.exit_json(**result)
 

lib/ansible/modules/network/junos/junos_static_route.py

@@ -138,15 +138,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_system.py

@@ -109,15 +109,10 @@ diff.prepared:
 import collections
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_user.py

@@ -140,15 +140,15 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec, get_connection
+from ansible.module_utils.network.junos.junos import junos_argument_spec, get_connection, tostring
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes
 from ansible.module_utils.network.junos.junos import load_config, locked_config
 from ansible.module_utils.six import iteritems
 
 try:
-    from lxml.etree import Element, SubElement, tostring
+    from lxml.etree import Element, SubElement
 except ImportError:
-    from xml.etree.ElementTree import Element, SubElement, tostring
+    from xml.etree.ElementTree import Element, SubElement
 
 ROLES = ['operator', 'read-only', 'super-user', 'unauthorized']
 USE_PERSISTENT_CONNECTION = True

lib/ansible/modules/network/junos/junos_vlan.py

@@ -115,15 +115,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/modules/network/junos/junos_vrf.py

@@ -173,15 +173,10 @@ from copy import deepcopy
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.network.common.utils import remove_default_spec
-from ansible.module_utils.network.junos.junos import junos_argument_spec
+from ansible.module_utils.network.junos.junos import junos_argument_spec, tostring
 from ansible.module_utils.network.junos.junos import load_config, map_params_to_obj, map_obj_to_ele, to_param_list
 from ansible.module_utils.network.junos.junos import commit_configuration, discard_changes, locked_config
 
-try:
-    from lxml.etree import tostring
-except ImportError:
-    from xml.etree.ElementTree import tostring
-
 USE_PERSISTENT_CONNECTION = True
 
 

lib/ansible/plugins/netconf/iosxr.py

@@ -26,7 +26,7 @@ import collections
 
 from ansible import constants as C
 from ansible.module_utils.network.common.netconf import remove_namespaces
-from ansible.module_utils.network.iosxr.iosxr import build_xml
+from ansible.module_utils.network.iosxr.iosxr import build_xml, etree_find
 from ansible.errors import AnsibleConnectionFailure, AnsibleError
 from ansible.plugins.netconf import NetconfBase
 from ansible.plugins.netconf import ensure_connected
@@ -64,10 +64,10 @@ class Netconf(NetconfBase):
         install_filter = build_xml('install', install_meta, opcode='filter')
 
         reply = self.get(install_filter)
-        ele_boot_variable = etree.fromstring(reply).find('.//boot-variable/boot-variable')
+        ele_boot_variable = etree_find(reply, 'boot-variable/boot-variable')
         if ele_boot_variable is not None:
             device_info['network_os_image'] = re.split('[:|,]', ele_boot_variable.text)[1]
-        ele_package_name = etree.fromstring(reply).find('.//package-name')
+        ele_package_name = etree_find(reply, 'package-name')
         if ele_package_name is not None:
             device_info['network_os_package'] = ele_package_name.text
             device_info['network_os_version'] = re.split('-', ele_package_name.text)[-1]
@@ -75,7 +75,7 @@ class Netconf(NetconfBase):
         hostname_filter = build_xml('host-names', opcode='filter')
 
         reply = self.get(hostname_filter)
-        hostname_ele = etree.fromstring(reply).find('.//host-name')
+        hostname_ele = etree_find(reply, 'host-name')
         device_info['network_os_hostname'] = hostname_ele.text if hostname_ele is not None else None
 
         return device_info

lib/ansible/plugins/netconf/junos.py

@@ -73,7 +73,6 @@ class Netconf(NetconfBase):
         :target: is the name of the configuration datastore being edited
         :config: is the configuration in string format."""
         if kwargs.get('config'):
-            kwargs['config'] = to_bytes(kwargs['config'], errors='surrogate_or_strict')
             if kwargs.get('format', 'xml') == 'xml':
                 kwargs['config'] = to_ele(kwargs['config'])
 

test/units/modules/network/junos/test_junos_rpc.py

@@ -103,7 +103,7 @@ class TestJunosCommandModule(TestJunosModule):
         result = self.execute_module(format='xml')
         args, kwargs = self.exec_rpc.call_args
         reply = args[1]
-        self.assertTrue(reply.find(b'<interface>em0</interface><media /></get-software-information>'))
+        self.assertTrue(reply.find('<interface>em0</interface><media /></get-software-information>'))
 
     def test_junos_rpc_attrs(self):
         set_module_args(dict(rpc='load-configuration', output='xml', attrs={'url': '/var/tmp/config.conf'}))