From fc5d71de0da187658899cf12e0a6b3cdb210f5fc Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 21 Jul 2017 10:23:16 -0700 Subject: [PATCH] Fix for to_text and to_bytes error handlers * surrogate_then_strict doesn't exist. Switch to surrogate_or_strict instead. * Found some bugs in the _text.py implementation * The composed error handlers (error handlers which are made up of two or more python encoding error handlers) had a wrong string in it, 'surrogate_or_escape' doesn't exist. Replaced that with 'surrogate_or_replace' which is the correct handler name. * Left comment about the implicit conditions that are part of the surrogate_then_replace code path Fixes #23865 Fixes #23861 --- lib/ansible/module_utils/_text.py | 5 ++++- lib/ansible/module_utils/basic.py | 18 +++++++++--------- lib/ansible/plugins/filter/core.py | 6 +++--- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/ansible/module_utils/_text.py b/lib/ansible/module_utils/_text.py index 81943bbea8d..34d8f520240 100644 --- a/lib/ansible/module_utils/_text.py +++ b/lib/ansible/module_utils/_text.py @@ -44,7 +44,7 @@ except LookupError: HAS_SURROGATEESCAPE = False -_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_escape', +_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_replace', 'surrogate_or_strict', 'surrogate_then_replace')) @@ -133,6 +133,9 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): return obj.encode(encoding, errors) except UnicodeEncodeError: if original_errors in (None, 'surrogate_then_replace'): + # We should only reach this if encoding was non-utf8 original_errors was + # surrogate_then_escape and errors was surrogateescape + # Slow but works return_string = obj.encode('utf-8', 'surrogateescape') return_string = return_string.decode('utf-8', 'replace') diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index d353ba45558..39c118da30c 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1011,7 +1011,7 @@ class AnsibleModule(object): return context def user_and_group(self, path, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) st = os.lstat(b_path) @@ -1105,10 +1105,10 @@ class AnsibleModule(object): return changed def set_owner_if_different(self, path, owner, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') if owner is None: return changed orig_uid, orig_gid = self.user_and_group(path, expand) @@ -1139,10 +1139,10 @@ class AnsibleModule(object): return changed def set_group_if_different(self, path, group, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') if group is None: return changed orig_uid, orig_gid = self.user_and_group(b_path, expand) @@ -1173,10 +1173,10 @@ class AnsibleModule(object): return changed def set_mode_if_different(self, path, mode, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') path_stat = os.lstat(b_path) if mode is None: @@ -1253,10 +1253,10 @@ class AnsibleModule(object): if attributes is None: return changed - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') existing = self.get_file_attributes(b_path) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 995688989e0..4d7b2b25db0 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -262,7 +262,7 @@ def get_hash(data, hashtype='sha1'): except: return None - h.update(to_bytes(data, errors='surrogate_then_strict')) + h.update(to_bytes(data, errors='surrogate_or_strict')) return h.hexdigest() @@ -453,11 +453,11 @@ def do_groupby(environment, value, attribute): def b64encode(string): - return to_text(base64.b64encode(to_bytes(string, errors='surrogate_then_strict'))) + return to_text(base64.b64encode(to_bytes(string, errors='surrogate_or_strict'))) def b64decode(string): - return to_text(base64.b64decode(to_bytes(string, errors='surrogate_then_strict'))) + return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict'))) class FilterModule(object):