Migrate basestring to a python3 compatible type (#17199)

This commit is contained in:
Toshio Kuratomi 2016-08-23 13:13:44 -07:00 committed by GitHub
parent a695e18615
commit a22909c226
28 changed files with 137 additions and 101 deletions

View file

@ -277,7 +277,8 @@ To test if something is a string, consider that it may be unicode.
if type(x) == str:
# yes
if isinstance(x, basestring):
from ansible.compat.six import string_types
if isinstance(x, string_types):
Cleverness
==========

View file

@ -148,6 +148,7 @@ from pynsot.client import get_api_client
from pynsot.app import HttpServerError
from click.exceptions import UsageError
from six import string_types
def warning(*objs):
print("WARNING: ", *objs, file=sys.stderr)
@ -251,7 +252,7 @@ class NSoTInventory(object):
obj[group]['hosts'] = []
obj[group]['vars'] = hostvars
try:
assert isinstance(query, basestring)
assert isinstance(query, string_types)
except:
sys.exit('ERR: Group queries must be a single string\n'
' Group: %s\n'

View file

@ -40,7 +40,7 @@ import sys
import time
import ConfigParser
from six import text_type
from six import text_type, string_types
# Disable logging message trigged by pSphere/suds.
try:
@ -160,7 +160,7 @@ class VMwareInventory(object):
if isinstance(v, collections.MutableMapping):
items.extend(self._flatten_dict(v, new_key, sep).items())
elif isinstance(v, (list, tuple)):
if all([isinstance(x, basestring) for x in v]):
if all([isinstance(x, string_types) for x in v]):
items.append((new_key, v))
else:
items.append((new_key, v))
@ -208,7 +208,7 @@ class VMwareInventory(object):
if obj_info != ():
l.append(obj_info)
return l
elif isinstance(obj, (type(None), bool, int, long, float, basestring)):
elif isinstance(obj, (type(None), bool, int, long, float, string_types)):
return obj
else:
return ()

View file

@ -24,7 +24,7 @@ import os
import traceback
import textwrap
from ansible.compat.six import iteritems
from ansible.compat.six import iteritems, string_types
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleOptionsError
@ -309,13 +309,13 @@ class DocCLI(CLI):
maintainers = set()
if 'author' in doc:
if isinstance(doc['author'], basestring):
if isinstance(doc['author'], string_types):
maintainers.add(doc['author'])
else:
maintainers.update(doc['author'])
if 'maintainers' in doc:
if isinstance(doc['maintainers'], basestring):
if isinstance(doc['maintainers'], string_types):
maintainers.add(doc['author'])
else:
maintainers.update(doc['author'])

View file

@ -30,6 +30,7 @@ import urllib
from urllib2 import quote as urlquote, HTTPError
import ansible.constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.galaxy.token import GalaxyToken
@ -41,22 +42,24 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
def g_connect(method):
''' wrapper to lazily initialize connection info to galaxy '''
def wrapped(self, *args, **kwargs):
if not self.initialized:
display.vvvv("Initial connection to galaxy_server: %s" % self._api_server)
server_version = self._get_server_api_version()
if not server_version in self.SUPPORTED_VERSIONS:
if server_version not in self.SUPPORTED_VERSIONS:
raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version)
self.baseurl = '%s/api/%s' % (self._api_server, server_version)
self.version = server_version # for future use
self.version = server_version # for future use
display.vvvv("Base API: %s" % self.baseurl)
self.initialized = True
return method(self, *args, **kwargs)
return wrapped
class GalaxyAPI(object):
''' This class is meant to be used as a API client for an Ansible Galaxy server '''
@ -77,7 +80,6 @@ class GalaxyAPI(object):
if galaxy.options.api_server != C.GALAXY_SERVER:
self._api_server = galaxy.options.api_server
def __auth_header(self):
token = self.token.get()
if token is None:
@ -112,7 +114,7 @@ class GalaxyAPI(object):
"""
url = '%s/api/' % self._api_server
try:
return_data =open_url(url, validate_certs=self._validate_certs)
return_data = open_url(url, validate_certs=self._validate_certs)
except Exception as e:
raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_str(e)))
@ -121,7 +123,7 @@ class GalaxyAPI(object):
except Exception as e:
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_str(e)))
if not 'current_version' in data:
if 'current_version' not in data:
raise AnsibleError("missing required 'current_version' from server response (%s)" % url)
return data['current_version']
@ -159,9 +161,9 @@ class GalaxyAPI(object):
Check the status of an import task.
"""
url = '%s/imports/' % self.baseurl
if not task_id is None:
if task_id is not None:
url = "%s?id=%d" % (url,task_id)
elif not github_user is None and not github_repo is None:
elif github_user is not None and github_repo is not None:
url = "%s?github_user=%s&github_repo=%s" % (url,github_user,github_repo)
else:
raise AnsibleError("Expected task_id or github_user and github_repo")
@ -249,11 +251,11 @@ class GalaxyAPI(object):
page_size = kwargs.get('page_size', None)
author = kwargs.get('author', None)
if tags and isinstance(tags, basestring):
if tags and isinstance(tags, string_types):
tags = tags.split(',')
search_url += '&tags_autocomplete=' + '+'.join(tags)
if platforms and isinstance(platforms, basestring):
if platforms and isinstance(platforms, string_types):
platforms = platforms.split(',')
search_url += '&platforms_autocomplete=' + '+'.join(platforms)

View file

@ -133,54 +133,10 @@ except ImportError:
except ImportError:
pass
try:
# Python 2.6+
from ast import literal_eval
except ImportError:
# a replacement for literal_eval that works with python 2.4. from:
# https://mail.python.org/pipermail/python-list/2009-September/551880.html
# which is essentially a cut/paste from an earlier (2.6) version of python's
# ast.py
from compiler import ast, parse
def literal_eval(node_or_string):
"""
Safely evaluate an expression node or a string containing a Python
expression. The string or node provided may only consist of the following
Python literal structures: strings, numbers, tuples, lists, dicts, booleans,
and None.
"""
_safe_names = {'None': None, 'True': True, 'False': False}
# Okay to use basestring and long here because this is only for
# python 2.4 and 2.5
if isinstance(node_or_string, basestring):
node_or_string = parse(node_or_string, mode='eval')
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.node
def _convert(node):
if isinstance(node, ast.Const) and isinstance(node.value, (basestring, int, float, long, complex)):
return node.value
elif isinstance(node, ast.Tuple):
return tuple(map(_convert, node.nodes))
elif isinstance(node, ast.List):
return list(map(_convert, node.nodes))
elif isinstance(node, ast.Dict):
return dict((_convert(k), _convert(v)) for k, v in node.items())
elif isinstance(node, ast.Name):
if node.name in _safe_names:
return _safe_names[node.name]
elif isinstance(node, ast.UnarySub):
return -_convert(node.expr)
raise ValueError('malformed string')
return _convert(node_or_string)
_literal_eval = literal_eval
from ansible.module_utils.pycompat24 import get_exception, literal_eval
from ansible.module_utils.six import (PY2, PY3, b, binary_type, integer_types,
iteritems, text_type, string_types)
from ansible.module_utils.six.moves import map, reduce
from ansible.module_utils.pycompat24 import get_exception
from ansible.module_utils._text import to_native
_NUMBERTYPES = tuple(list(integer_types) + [float])
@ -213,6 +169,8 @@ except NameError:
# Python 3
basestring = string_types
_literal_eval = literal_eval
# End of deprecated names
# Internal global holding passed in params. This is consulted in case

View file

@ -49,6 +49,7 @@ try:
except:
HAS_LOOSE_VERSION = False
from ansible.module_utils.six import string_types
class AnsibleAWSError(Exception):
pass
@ -343,7 +344,7 @@ def ansible_dict_to_boto3_filter_list(filters_dict):
filters_list = []
for k,v in filters_dict.iteritems():
filter_dict = {'Name': k}
if isinstance(v, basestring):
if isinstance(v, string_types):
filter_dict['Values'] = [v]
else:
filter_dict['Values'] = v
@ -438,7 +439,7 @@ def get_ec2_security_group_ids_from_names(sec_group_list, ec2_connection, vpc_id
sec_group_id_list = []
if isinstance(sec_group_list, basestring):
if isinstance(sec_group_list, string_types):
sec_group_list = [sec_group_list]
# Get all security groups

View file

@ -22,6 +22,7 @@ from distutils.version import LooseVersion
from ansible.module_utils.basic import AnsibleModule, env_fallback, get_exception
from ansible.module_utils.shell import Shell, ShellError, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
from ansible.module_utils.six import string_types
try:
from jnpr.junos import Device
@ -70,7 +71,7 @@ def to_list(val):
def xml_to_json(val):
if isinstance(val, basestring):
if isinstance(val, string_types):
return jxmlease.parse(val)
else:
return jxmlease.parse_etree(val)

View file

@ -33,6 +33,7 @@ import shlex
import itertools
from ansible.module_utils.basic import BOOLEANS_TRUE, BOOLEANS_FALSE
from ansible.module_utils.six import string_types
DEFAULT_COMMENT_TOKENS = ['#', '!', '/*', '*/']
@ -197,7 +198,7 @@ class NetworkConfig(object):
self.load(open(filename).read())
def get(self, path):
if isinstance(path, basestring):
if isinstance(path, string_types):
path = [path]
for item in self._config:
if item.text == path[-1]:

View file

@ -32,6 +32,7 @@ import itertools
import shlex
from ansible.module_utils.basic import BOOLEANS_TRUE, BOOLEANS_FALSE
from ansible.module_utils.six import string_types
def to_list(val):
if isinstance(val, (list, tuple)):
@ -75,7 +76,7 @@ class Cli(object):
elif isinstance(command, dict):
output = cmd.get('output') or output
cmd = cmd['command']
if isinstance(prompt, basestring):
if isinstance(prompt, string_types):
prompt = re.compile(re.escape(prompt))
return Command(command, output, prompt=prompt, response=response)

View file

@ -42,3 +42,47 @@ def get_exception():
"""
return sys.exc_info()[1]
try:
# Python 2.6+
from ast import literal_eval
except ImportError:
# a replacement for literal_eval that works with python 2.4. from:
# https://mail.python.org/pipermail/python-list/2009-September/551880.html
# which is essentially a cut/paste from an earlier (2.6) version of python's
# ast.py
from compiler import ast, parse
from ansible.module_utils.six import binary_type, string_types, text_type
def literal_eval(node_or_string):
"""
Safely evaluate an expression node or a string containing a Python
expression. The string or node provided may only consist of the following
Python literal structures: strings, numbers, tuples, lists, dicts, booleans,
and None.
"""
_safe_names = {'None': None, 'True': True, 'False': False}
if isinstance(node_or_string, string_types):
node_or_string = parse(node_or_string, mode='eval')
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.node
def _convert(node):
# Okay to use long here because this is only for python 2.4 and 2.5
if isinstance(node, ast.Const) and isinstance(node.value, (text_type, binary_type, int, float, long, complex)):
return node.value
elif isinstance(node, ast.Tuple):
return tuple(map(_convert, node.nodes))
elif isinstance(node, ast.List):
return list(map(_convert, node.nodes))
elif isinstance(node, ast.Dict):
return dict((_convert(k), _convert(v)) for k, v in node.items())
elif isinstance(node, ast.Name):
if node.name in _safe_names:
return _safe_names[node.name]
elif isinstance(node, ast.UnarySub):
return -_convert(node.expr)
raise ValueError('malformed string')
return _convert(node_or_string)
__all__ = ('get_exception', 'literal_eval')

View file

@ -33,6 +33,7 @@ import re
from uuid import UUID
from ansible.module_utils.basic import BOOLEANS
from ansible.module_utils.six import text_type, binary_type
FINAL_STATUSES = ('ACTIVE', 'ERROR')
VOLUME_STATUS = ('available', 'attaching', 'creating', 'deleting', 'in-use',
@ -44,7 +45,7 @@ CLB_PROTOCOLS = ['DNS_TCP', 'DNS_UDP', 'FTP', 'HTTP', 'HTTPS', 'IMAPS',
'IMAPv4', 'LDAP', 'LDAPS', 'MYSQL', 'POP3', 'POP3S', 'SMTP',
'TCP', 'TCP_CLIENT_FIRST', 'UDP', 'UDP_STREAM', 'SFTP']
NON_CALLABLES = (basestring, bool, dict, int, list, type(None))
NON_CALLABLES = (text_type, binary_type, bool, dict, int, list, type(None))
PUBLIC_NET_ID = "00000000-0000-0000-0000-000000000000"
SERVICE_NET_ID = "11111111-1111-1111-1111-111111111111"

View file

@ -34,6 +34,8 @@ import select
import subprocess
import json
from ansible.module_utils.six import text_type, binary_type
class Service(object):
"""
This is the generic Service manipulation class that is subclassed based on system.
@ -112,7 +114,7 @@ class Service(object):
os._exit(0)
# Start the command
if isinstance(cmd, basestring):
if isinstance(cmd, (text_type, binary_type)):
cmd = shlex.split(cmd)
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=lambda: os.close(pipe[1]))
stdout = ""

View file

@ -37,7 +37,7 @@ class Taggable:
def _load_tags(self, attr, ds):
if isinstance(ds, list):
return ds
elif isinstance(ds, basestring):
elif isinstance(ds, string_types):
value = ds.split(',')
if isinstance(value, list):
return [ x.strip() for x in value ]

View file

@ -21,6 +21,7 @@ __metaclass__ = type
import json
from ansible.compat.six import string_types
from ansible.plugins.action import ActionBase
from ansible.plugins.action.net_template import ActionModule as NetActionModule
@ -32,7 +33,7 @@ class ActionModule(NetActionModule, ActionBase):
result = dict(changed=False)
if isinstance(self._task.args['src'], basestring):
if isinstance(self._task.args['src'], string_types):
self._handle_template()
result.update(self._execute_module(module_name=self._task.action,

View file

@ -18,13 +18,12 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six import iteritems
from ansible.compat.six import iteritems, string_types
from ansible.plugins.action import ActionBase
from ansible.utils.boolean import boolean
from ansible.utils.vars import isidentifier
class ActionModule(ActionBase):
TRANSFERS_FILES = False
@ -45,7 +44,7 @@ class ActionModule(ActionBase):
result['msg'] = "The variable name '%s' is not valid. Variables must start with a letter or underscore character, and contain only letters, numbers and underscores." % k
return result
if isinstance(v, basestring) and v.lower() in ('true', 'false', 'yes', 'no'):
if isinstance(v, string_types) and v.lower() in ('true', 'false', 'yes', 'no'):
v = boolean(v)
facts[k] = v

View file

@ -24,6 +24,7 @@ import os
import smtplib
import json
from ansible.compat.six import string_types
from ansible.utils.unicode import to_bytes
from ansible.plugins.callback import CallbackBase
@ -108,7 +109,7 @@ class CallbackModule(CallbackBase):
res = result._result
sender = '"Ansible: %s" <root>' % host
if isinstance(res, basestring):
if isinstance(res, string_types):
subject = 'Unreachable: %s' % res.strip('\r\n').split('\n')[-1]
body = 'An error occurred for host ' + host + ' with the following message:\n\n' + res
else:
@ -123,7 +124,7 @@ class CallbackModule(CallbackBase):
res = result._result
sender = '"Ansible: %s" <root>' % host
if isinstance(res, basestring):
if isinstance(res, string_types):
subject = 'Async failure: %s' % res.strip('\r\n').split('\n')[-1]
body = 'An error occurred for host ' + host + ' with the following message:\n\n' + res
else:

View file

@ -30,6 +30,7 @@ from functools import wraps
from ansible.compat.six import with_metaclass
from ansible import constants as C
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.plugins import shell_loader
from ansible.utils.unicode import to_bytes, to_unicode
@ -248,7 +249,7 @@ class ConnectionBase(with_metaclass(ABCMeta, object)):
def check_password_prompt(self, output):
if self._play_context.prompt is None:
return False
elif isinstance(self._play_context.prompt, basestring):
elif isinstance(self._play_context.prompt, string_types):
return output.startswith(self._play_context.prompt)
else:
return self._play_context.prompt(output)

View file

@ -26,7 +26,7 @@ import shlex
import traceback
import json
from ansible.compat.six import string_types
from ansible.compat.six.moves.urllib.parse import urlunsplit
from ansible.errors import AnsibleError, AnsibleConnectionFailure
@ -106,7 +106,7 @@ class Connection(ConnectionBase):
else:
self._winrm_transport = transport_selector
self._winrm_transport = hostvars.get('ansible_winrm_transport', self._winrm_transport)
if isinstance(self._winrm_transport, basestring):
if isinstance(self._winrm_transport, string_types):
self._winrm_transport = [x.strip() for x in self._winrm_transport.split(',') if x.strip()]
unsupported_transports = set(self._winrm_transport).difference(self._winrm_supported_authtypes)

View file

@ -126,8 +126,7 @@ def fileglob(pathname):
def regex_replace(value='', pattern='', replacement='', ignorecase=False):
''' Perform a `re.sub` returning a string '''
if not isinstance(value, basestring):
value = str(value)
value = to_unicode(value, errors='strict', nonstring='simplerepr')
if ignorecase:
flags = re.I

View file

@ -122,6 +122,7 @@ import os
from jinja2.exceptions import UndefinedError
from ansible.compat.six import string_types
from ansible.errors import AnsibleFileNotFound, AnsibleLookupError, AnsibleUndefinedVariable
from ansible.plugins.lookup import LookupBase
from ansible.utils.boolean import boolean
@ -146,14 +147,14 @@ class LookupModule(LookupBase):
skip = boolean(term.get('skip', False))
filelist = files
if isinstance(files, basestring):
if isinstance(files, string_types):
files = files.replace(',', ' ')
files = files.replace(';', ' ')
filelist = files.split(' ')
pathlist = paths
if paths:
if isinstance(paths, basestring):
if isinstance(paths, string_types):
paths = paths.replace(',', ' ')
paths = paths.replace(':', ' ')
paths = paths.replace(';', ' ')

View file

@ -17,6 +17,7 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms
@ -44,7 +45,7 @@ class LookupModule(LookupBase):
# ignore undefined items
break
if isinstance(term, basestring):
if isinstance(term, string_types):
# convert a variable to a list
term2 = listify_lookup_plugin_terms(term, templar=self._templar, loader=self._loader)
# but avoid converting a plain string to a list of one string

View file

@ -17,6 +17,7 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.utils.listify import listify_lookup_plugin_terms
@ -41,7 +42,7 @@ class LookupModule(LookupBase):
_raise_terms_error()
# first term should be a list (or dict), second a string holding the subkey
if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], basestring):
if not isinstance(terms[0], (list, dict)) or not isinstance(terms[1], string_types):
_raise_terms_error("first a dict or a list, second a string pointing to the subkey")
subelements = terms[1].split(".")
@ -59,7 +60,7 @@ class LookupModule(LookupBase):
flags = {}
if len(terms) == 3:
flags = terms[2]
if not isinstance(flags, dict) and not all([isinstance(key, basestring) and key in FLAGS for key in flags]):
if not isinstance(flags, dict) and not all([isinstance(key, string_types) and key in FLAGS for key in flags]):
_raise_terms_error("the optional third item must be a dict with flags %s" % FLAGS)
# build_items

View file

@ -20,8 +20,6 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes
# Note, sha1 is the only hash algorithm compatible with python2.4 and with
# FIPS-140 mode (as of 11-2014)
@ -40,12 +38,16 @@ except ImportError:
# Assume we're running in FIPS mode here
_md5 = None
from ansible.compat.six import string_types
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes
def secure_hash_s(data, hash_func=sha1):
''' Return a secure hash hex digest of data. '''
digest = hash_func()
try:
if not isinstance(data, basestring):
if not isinstance(data, string_types):
data = "%s" % data
digest.update(data)
except UnicodeEncodeError:

View file

@ -24,10 +24,12 @@ __metaclass__ = type
import os
import sys
import ast
from ansible.parsing.yaml.loader import AnsibleLoader
import traceback
from collections import MutableMapping, MutableSet, MutableSequence
from ansible.compat.six import string_types
from ansible.parsing.yaml.loader import AnsibleLoader
from ansible.plugins import fragment_loader
try:
@ -76,7 +78,7 @@ def get_docstring(filename, verbose=False):
doc = AnsibleLoader(child.value.s, file_name=filename).get_single_data()
fragments = doc.get('extends_documentation_fragment', [])
if isinstance(fragments, basestring):
if isinstance(fragments, string_types):
fragments = [ fragments ]
# Allow the module to specify a var other than DOCUMENTATION

View file

@ -37,9 +37,6 @@ _LATIN1_ALIASES = frozenset(('latin-1', 'LATIN-1', 'latin1', 'LATIN1',
# EXCEPTION_CONVERTERS is defined below due to using to_unicode
if PY3:
basestring = (str, bytes)
def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None):
'''Convert an object into a :class:`unicode` string
@ -93,9 +90,9 @@ def to_unicode(obj, encoding='utf-8', errors='replace', nonstring=None):
'''
# Could use isbasestring/isunicode here but we want this code to be as
# fast as possible
if isinstance(obj, basestring):
if isinstance(obj, text_type):
return obj
if isinstance(obj, text_type):
return obj
if isinstance(obj, binary_type):
if encoding in _UTF8_ALIASES:
return text_type(obj, 'utf-8', errors)
if encoding in _LATIN1_ALIASES:
@ -202,9 +199,9 @@ def to_bytes(obj, encoding='utf-8', errors='replace', nonstring=None):
'''
# Could use isbasestring, isbytestring here but we want this to be as fast
# as possible
if isinstance(obj, basestring):
if isinstance(obj, binary_type):
return obj
if isinstance(obj, binary_type):
return obj
if isinstance(obj, text_type):
return obj.encode(encoding, errors)
if not nonstring:
nonstring = 'simplerepr'

View file

@ -0,0 +1,16 @@
#!/bin/sh
BASEDIR=${1-"."}
# Not entirely correct but
# * basestring is still present and harmless in comments
# * basestring is also currently present in modules. Porting of modules is more
# of an Ansible 2.3 or greater goal.
BASESTRING_USERS=$(grep -r basestring $BASEDIR |grep isinstance| grep -v lib/ansible/compat/six/_six.py|grep -v lib/ansible/module_utils/six.py|grep -v lib/ansible/modules/core|grep -v lib/ansible/modules/extras)
if test -n "$BASESTRING_USERS" ; then
printf "$BASESTRING_USERS"
exit 1
else
exit 0
fi

View file

@ -3,16 +3,18 @@ import mock
import os
import re
from nose.tools import eq_
try:
from nose.tools import assert_raises_regexp
except ImportError:
from ansible.compat.six import string_types
# Python < 2.7
def assert_raises_regexp(expected, regexp, callable, *a, **kw):
try:
callable(*a, **kw)
except expected as e:
if isinstance(regexp, basestring):
if isinstance(regexp, string_types):
regexp = re.compile(regexp)
if not regexp.search(str(e)):
raise Exception('"%s" does not match "%s"' %