diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index 5f6a818d7fe..72c33006ca7 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -60,18 +60,18 @@ class BaseInventoryPlugin(object): def verify_file(self, path): ''' Verify if file is usable by this plugin, base does minimal accessability check ''' - b_path = to_bytes(path) + b_path = to_bytes(path, errors='surrogate_or_strict') return (os.path.exists(b_path) and os.access(b_path, os.R_OK)) def get_cache_prefix(self, path): ''' create predictable unique prefix for plugin/inventory ''' m = hashlib.sha1() - m.update(to_bytes(self.NAME)) + m.update(to_bytes(self.NAME, errors='surrogate_or_strict')) d1 = m.hexdigest() n = hashlib.sha1() - n.update(to_bytes(path)) + n.update(to_bytes(path, errors='surrogate_or_strict')) d2 = n.hexdigest() return 's_'.join([d1[:5], d2[:5]]) diff --git a/lib/ansible/plugins/inventory/advanced_host_list.py b/lib/ansible/plugins/inventory/advanced_host_list.py index 1783efa9615..5e4711f9b9f 100644 --- a/lib/ansible/plugins/inventory/advanced_host_list.py +++ b/lib/ansible/plugins/inventory/advanced_host_list.py @@ -36,9 +36,9 @@ class InventoryModule(BaseInventoryPlugin): def verify_file(self, host_list): valid = False - b_path = to_bytes(host_list) + b_path = to_bytes(host_list, errors='surrogate_or_strict') if not os.path.exists(b_path) and ',' in host_list: - valid = True + valid = True return valid def parse(self, inventory, loader, host_list, cache=True): @@ -61,7 +61,7 @@ class InventoryModule(BaseInventoryPlugin): if host not in self.inventory.hosts: self.inventory.add_host(host, group='ungrouped', port=port) except Exception as e: - raise AnsibleParserError("Invalid data from string, could not parse: %s" % str(e)) + raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e)) def _expand_hostpattern(self, hostpattern): ''' diff --git a/lib/ansible/plugins/inventory/host_list.py b/lib/ansible/plugins/inventory/host_list.py index b9bfa9dbd5c..6c166349167 100644 --- a/lib/ansible/plugins/inventory/host_list.py +++ b/lib/ansible/plugins/inventory/host_list.py @@ -27,8 +27,7 @@ EXAMPLES = r''' import os from ansible.errors import AnsibleError, AnsibleParserError -from ansible.module_utils.six import string_types -from ansible.module_utils._text import to_bytes, to_text, to_native +from ansible.module_utils._text import to_bytes, to_native from ansible.parsing.utils.addresses import parse_address from ansible.plugins.inventory import BaseInventoryPlugin @@ -40,9 +39,9 @@ class InventoryModule(BaseInventoryPlugin): def verify_file(self, host_list): valid = False - b_path = to_bytes(host_list) + b_path = to_bytes(host_list, errors='surrogate_or_strict') if not os.path.exists(b_path) and ',' in host_list: - valid = True + valid = True return valid def parse(self, inventory, loader, host_list, cache=True): @@ -64,4 +63,4 @@ class InventoryModule(BaseInventoryPlugin): if host not in self.inventory.hosts: self.inventory.add_host(host, group='ungrouped', port=port) except Exception as e: - raise AnsibleParserError("Invalid data from string, could not parse: %s" % str(e)) + raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e)) diff --git a/lib/ansible/plugins/inventory/ini.py b/lib/ansible/plugins/inventory/ini.py index 8033ccf3151..68d0380b2d9 100644 --- a/lib/ansible/plugins/inventory/ini.py +++ b/lib/ansible/plugins/inventory/ini.py @@ -105,7 +105,7 @@ class InventoryModule(BaseFileInventoryPlugin): if self.loader: (b_data, private) = self.loader._get_file_contents(path) else: - b_path = to_bytes(path) + b_path = to_bytes(path, errors='surrogate_or_strict') with open(b_path, 'rb') as fh: b_data = fh.read() @@ -366,14 +366,14 @@ class InventoryModule(BaseFileInventoryPlugin): # [naughty:children] # only get coal in their stockings self.patterns['section'] = re.compile( - r'''^\[ + to_text(r'''^\[ ([^:\]\s]+) # group name (see groupname below) (?::(\w+))? # optional : and tag name \] \s* # ignore trailing whitespace (?:\#.*)? # and/or a comment till the $ # end of the line - ''', re.X + ''', errors='surrogate_or_strict'), re.X ) # FIXME: What are the real restrictions on group names, or rather, what @@ -382,10 +382,10 @@ class InventoryModule(BaseFileInventoryPlugin): # precise rules in order to support better diagnostics. self.patterns['groupname'] = re.compile( - r'''^ + to_text(r'''^ ([^:\]\s]+) \s* # ignore trailing whitespace (?:\#.*)? # and/or a comment till the $ # end of the line - ''', re.X + ''', errors='surrogate_or_strict'), re.X ) diff --git a/lib/ansible/plugins/inventory/virtualbox.py b/lib/ansible/plugins/inventory/virtualbox.py index 9efb74da777..1b80945b913 100644 --- a/lib/ansible/plugins/inventory/virtualbox.py +++ b/lib/ansible/plugins/inventory/virtualbox.py @@ -49,10 +49,11 @@ simple_config_file: import os +from collections import MutableMapping from subprocess import Popen, PIPE from ansible.errors import AnsibleParserError -from ansible.module_utils._text import to_bytes, to_text +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.plugins.inventory import BaseInventoryPlugin @@ -60,14 +61,14 @@ class InventoryModule(BaseInventoryPlugin): ''' Host inventory parser for ansible using local virtualbox. ''' NAME = 'virtualbox' - VBOX = "VBoxManage" + VBOX = b"VBoxManage" def _query_vbox_data(self, host, property_path): ret = None try: - cmd = [self.VBOX, 'guestproperty', 'get', host, property_path] + cmd = [self.VBOX, b'guestproperty', b'get', to_bytes(host, errors='surrogate_or_strict'), to_bytes(property_path, errors='surrogate_or_strict')] x = Popen(cmd, stdout=PIPE) - ipinfo = x.stdout.read() + ipinfo = to_text(x.stdout.read(), errors='surrogate_or_strict') if 'Value' in ipinfo: a, ip = ipinfo.split(':', 1) ret = ip.strip() @@ -81,7 +82,7 @@ class InventoryModule(BaseInventoryPlugin): for host in hostvars: # create vars from vbox properties - if data.get('query') and isinstance(data['query'], dict): + if data.get('query') and isinstance(data['query'], MutableMapping): for varname in data['query']: hostvars[host][varname] = self._query_vbox_data(host, data['query'][varname]) @@ -168,7 +169,7 @@ class InventoryModule(BaseInventoryPlugin): try: config_data = self.loader.load_from_file(path) except Exception as e: - raise AnsibleParserError(e) + raise AnsibleParserError(to_native(e)) if not config_data or config_data.get('plugin') != self.NAME: # this is not my config file @@ -182,26 +183,26 @@ class InventoryModule(BaseInventoryPlugin): pass if not source_data: - pwfile = to_bytes(config_data.get('settings_password_file')) + b_pwfile = to_bytes(config_data.get('settings_password_file'), errors='surrogate_or_strict') running = config_data.get('running_only', False) # start getting data - cmd = [self.VBOX, 'list', '-l'] + cmd = [self.VBOX, b'list', b'-l'] if running: - cmd.append('runningvms') + cmd.append(b'runningvms') else: - cmd.append('vms') + cmd.append(b'vms') - if pwfile and os.path.exists(pwfile): - cmd.append('--settingspwfile') - cmd.append(pwfile) + if b_pwfile and os.path.exists(b_pwfile): + cmd.append(b'--settingspwfile') + cmd.append(b_pwfile) try: p = Popen(cmd, stdout=PIPE) except Exception as e: - AnsibleParserError(e) + AnsibleParserError(to_native(e)) - source_data = p.stdout.readlines() - inventory.cache[cache_key] = to_text(source_data) + source_data = p.stdout.read() + inventory.cache[cache_key] = to_text(source_data, errors='surrogate_or_strict') - self._populate_from_source(source_data, config_data) + self._populate_from_source(source_data.splitlines(), config_data) diff --git a/lib/ansible/plugins/inventory/yaml.py b/lib/ansible/plugins/inventory/yaml.py index 560d25d96bc..8d5c457341c 100644 --- a/lib/ansible/plugins/inventory/yaml.py +++ b/lib/ansible/plugins/inventory/yaml.py @@ -51,11 +51,12 @@ all: # keys must be unique, i.e. only one 'hosts' per group import re import os +from collections import MutableMapping from ansible import constants as C from ansible.errors import AnsibleParserError from ansible.module_utils.six import string_types -from ansible.module_utils._text import to_bytes, to_text +from ansible.module_utils._text import to_native from ansible.parsing.utils.addresses import parse_address from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range @@ -74,9 +75,8 @@ class InventoryModule(BaseFileInventoryPlugin): def verify_file(self, path): valid = False - b_path = to_bytes(path) - if super(InventoryModule, self).verify_file(b_path): - file_name, ext = os.path.splitext(b_path) + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) if ext and ext in C.YAML_FILENAME_EXTENSIONS: valid = True return valid @@ -96,11 +96,11 @@ class InventoryModule(BaseFileInventoryPlugin): # We expect top level keys to correspond to groups, iterate over them # to get host, vars and subgroups (which we iterate over recursivelly) - if isinstance(data, dict): + if isinstance(data, MutableMapping): for group_name in data: self._parse_group(group_name, data[group_name]) else: - raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % data) + raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(data)) def _parse_group(self, group, group_data): @@ -108,7 +108,7 @@ class InventoryModule(BaseFileInventoryPlugin): self.inventory.add_group(group) - if isinstance(group_data, dict): + if isinstance(group_data, MutableMapping): # make sure they are dicts for section in ['vars', 'children', 'hosts']: if section in group_data and isinstance(group_data[section], string_types): @@ -167,4 +167,4 @@ class InventoryModule(BaseFileInventoryPlugin): ''' Compiles the regular expressions required to parse the inventory and stores them in self.patterns. ''' - self.patterns['groupname'] = re.compile(r'''^[A-Za-z_][A-Za-z0-9_]*$''') + self.patterns['groupname'] = re.compile(u'''^[A-Za-z_][A-Za-z0-9_]*$''')