cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[stable-2.7] Migrate from MySQLdb to PyMySQL (#40123)

Browse source 
* Migrate from MySQLdb to PyMySQL

* Deduplicate driver loading and failure message

* Explain requirements

* Apply requirements docs change to proxysql too

* Add changelog.
(cherry picked from commit d34cf93f1a)

Co-authored-by: Daniel Speichert <DSpeichert@users.noreply.github.com>
This commit is contained in:
Daniel Speichert 2018-09-19 17:44:05 +02:00 committed by Toshio Kuratomi
parent 547be8c957
commit bc08cf12ea
20 changed files with 93 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
changelogs/fragments/mysql-migrate_to_pymysql.yaml Normal file
View file

@ -0,0 +1,2 @@
bugfixes:
- mysql_*, proxysql_* - PyMySQL (a pure-Python MySQL driver) is now a preferred dependency also supporting Python 3.X.

14
lib/ansible/module_utils/mysql.py
View file

@ -30,10 +30,14 @@
import os
try:
import MySQLdb
mysqldb_found = True
import pymysql as mysql_driver
except ImportError:
mysqldb_found = False
try:
import MySQLdb as mysql_driver
except ImportError:
mysql_driver = None
mysql_driver_fail_msg = 'The PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) module is required.'
def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None,
@ -69,8 +73,8 @@ def mysql_connect(module, login_user=None, login_password=None, config_file='',
if connect_timeout is not None:
config['connect_timeout'] = connect_timeout
db_connection = MySQLdb.connect(**config)
db_connection = mysql_driver.connect(**config)
if cursor_class is not None:
return db_connection.cursor(cursorclass=MySQLdb.cursors.DictCursor)
return db_connection.cursor(cursorclass=mysql_driver.cursors.DictCursor)
else:
return db_connection.cursor()

17
lib/ansible/modules/database/mysql/mysql_db.py
View file

@ -67,7 +67,9 @@ requirements:
- mysql (command line binary)
- mysqldump (command line binary)
notes:
- Requires the python-mysqldb package on the remote host, as well as mysql and mysqldump binaries.
- Requires the PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) package on the remote host,
as well as mysql and mysqldump binaries.
- This module is B(not idempotent) when I(state) is C(import), and will import the dump file each time if run more than once.
extends_documentation_fragment: mysql
'''
@ -106,16 +108,9 @@ import pipes
import subprocess
import traceback
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.database import mysql_quote_identifier
from ansible.module_utils.mysql import mysql_connect, mysqldb_found
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
@ -278,8 +273,8 @@ def main():
supports_check_mode=True
)
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
db = module.params["name"]
encoding = module.params["encoding"]

19
lib/ansible/modules/database/mysql/mysql_replication.py
View file

@ -118,15 +118,8 @@ EXAMPLES = '''
import os
import warnings
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
@ -239,16 +232,16 @@ def main():
connect_timeout = module.params['connect_timeout']
config_file = module.params['config_file']
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
else:
warnings.filterwarnings('error', category=MySQLdb.Warning)
warnings.filterwarnings('error', category=mysql_driver.Warning)
login_password = module.params["login_password"]
login_user = module.params["login_user"]
try:
cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, None, 'MySQLdb.cursors.DictCursor',
cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, None, 'mysql_driver.cursors.DictCursor',
connect_timeout=connect_timeout)
except Exception as e:
if os.path.exists(config_file):
@ -325,7 +318,7 @@ def main():
chm.append("MASTER_AUTO_POSITION = 1")
try:
changemaster(cursor, chm, chm_params)
except MySQLdb.Warning as e:
except mysql_driver.Warning as e:
result['warning'] = to_native(e)
except Exception as e:
module.fail_json(msg='%s. Query == CHANGE MASTER TO %s' % (to_native(e), chm))

17
lib/ansible/modules/database/mysql/mysql_user.py
View file

@ -196,16 +196,9 @@ import re
import string
import traceback
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.database import SQLParseError
from ansible.module_utils.mysql import mysql_connect, mysqldb_found
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
@ -578,8 +571,8 @@ def main():
db = 'mysql'
sql_log_bin = module.params["sql_log_bin"]
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
cursor = None
try:
@ -618,14 +611,14 @@ def main():
else:
changed = user_mod(cursor, user, host, host_all, None, encrypted, priv, append_privs, module)
except (SQLParseError, InvalidPrivsError, MySQLdb.Error) as e:
except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
module.fail_json(msg=to_native(e), exception=traceback.format_exc())
else:
if host_all:
module.fail_json(msg="host_all parameter cannot be used when adding a user")
try:
changed = user_add(cursor, user, host, host_all, password, encrypted, priv, module.check_mode)
except (SQLParseError, InvalidPrivsError, MySQLdb.Error) as e:
except (SQLParseError, InvalidPrivsError, mysql_driver.Error) as e:
module.fail_json(msg=to_native(e), exception=traceback.format_exc())
elif state == "absent":
if user_exists(cursor, user, host, host_all):

15
lib/ansible/modules/database/mysql/mysql_variables.py
View file

@ -50,16 +50,9 @@ import os
import warnings
from re import match
try:
import MySQLdb
except ImportError:
mysqldb_found = False
else:
mysqldb_found = True
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.database import SQLParseError, mysql_quote_identifier
from ansible.module_utils.mysql import mysql_connect, mysqldb_found
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
@ -148,10 +141,10 @@ def main():
module.fail_json(msg="Cannot run without variable to operate with")
if match('^[0-9a-z_]+$', mysqlvar) is None:
module.fail_json(msg="invalid variable name \"%s\"" % mysqlvar)
if not mysqldb_found:
module.fail_json(msg="The MySQL-python module is required.")
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
else:
warnings.filterwarnings('error', category=MySQLdb.Warning)
warnings.filterwarnings('error', category=mysql_driver.Warning)
try:
cursor = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,

24
lib/ansible/modules/database/proxysql/proxysql_backend_servers.py
View file

@ -148,18 +148,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -192,10 +184,8 @@ def perform_checks(module):
msg="max_replication_lag must be set between 0 and 102400"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -457,8 +447,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -487,7 +477,7 @@ def main():
" and doesn't need to be updated.")
result['server'] = \
proxysql_server.get_server_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify server.. %s" % to_native(e)
)
@ -502,7 +492,7 @@ def main():
result['changed'] = False
result['msg'] = ("The server is already absent from the" +
" mysql_hosts memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove server.. %s" % to_native(e)
)

24
lib/ansible/modules/database/proxysql/proxysql_global_variables.py
View file

@ -73,17 +73,9 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -96,10 +88,8 @@ def perform_checks(module):
msg="login_port must be a valid unix port number (0-65535)"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(variable, cursor):
@ -211,8 +201,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -231,7 +221,7 @@ def main():
msg="The variable \"%s\" was not found" % variable
)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to get config.. %s" % to_native(e)
)
@ -264,7 +254,7 @@ def main():
msg="The variable \"%s\" was not found" % variable
)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to set config.. %s" % to_native(e)
)

19
lib/ansible/modules/database/proxysql/proxysql_manage_config.py
View file

@ -96,16 +96,9 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
try:
import MySQLdb
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -140,10 +133,8 @@ def perform_checks(module):
" with the CONFIG config_layer")
module.fail_json(msg=msg_string % module.params["direction"])
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def manage_config(manage_config_settings, cursor):
@ -201,7 +192,7 @@ def main():
login_user,
login_password,
config_file)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -214,7 +205,7 @@ def main():
try:
result['changed'] = manage_config(manage_config_settings,
cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to manage config.. %s" % to_native(e)
)

24
lib/ansible/modules/database/proxysql/proxysql_mysql_users.py
View file

@ -137,18 +137,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -161,10 +153,8 @@ def perform_checks(module):
msg="login_port must be a valid unix port number (0-65535)"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -424,8 +414,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -454,7 +444,7 @@ def main():
" and doesn't need to be updated.")
result['user'] = \
proxysql_user.get_user_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify user.. %s" % to_native(e)
)
@ -469,7 +459,7 @@ def main():
result['changed'] = False
result['msg'] = ("The user is already absent from the" +
" mysql_users memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove user.. %s" % to_native(e)
)

24
lib/ansible/modules/database/proxysql/proxysql_query_rules.py
View file

@ -221,18 +221,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -245,10 +237,8 @@ def perform_checks(module):
msg="login_port must be a valid unix port number (0-65535)"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -552,8 +542,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -583,7 +573,7 @@ def main():
result['rules'] = \
proxysql_query_rule.get_rule_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify rule.. %s" % to_native(e)
)
@ -606,7 +596,7 @@ def main():
result['changed'] = False
result['msg'] = ("The rule is already absent from the" +
" mysql_query_rules memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove rule.. %s" % to_native(e)
)

24
lib/ansible/modules/database/proxysql/proxysql_replication_hostgroups.py
View file

@ -95,17 +95,9 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -135,10 +127,8 @@ def perform_checks(module):
msg="reader_hostgroup cannot equal writer_hostgroup"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -328,8 +318,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -360,7 +350,7 @@ def main():
result['repl_group'] = \
proxysql_repl_group.get_repl_group_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify replication hostgroup.. %s" % to_native(e)
)
@ -378,7 +368,7 @@ def main():
" mysql_replication_hostgroups memory" +
" configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to delete replication hostgroup.. %s" % to_native(e)
)

24
lib/ansible/modules/database/proxysql/proxysql_scheduler.py
View file

@ -125,18 +125,10 @@ ANSIBLE_METADATA = {'metadata_version': '1.1',
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import mysql_connect
from ansible.module_utils.mysql import mysql_connect, mysql_driver, mysql_driver_fail_msg
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native
try:
import MySQLdb
import MySQLdb.cursors
except ImportError:
MYSQLDB_FOUND = False
else:
MYSQLDB_FOUND = True
# ===========================================
# proxysql module specific support methods.
#
@ -155,10 +147,8 @@ def perform_checks(module):
msg="interval_ms must between 100ms & 100000000ms"
)
if not MYSQLDB_FOUND:
module.fail_json(
msg="the python mysqldb module is required"
)
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
def save_config_to_disk(cursor):
@ -366,8 +356,8 @@ def main():
login_user,
login_password,
config_file,
cursor_class=MySQLdb.cursors.DictCursor)
except MySQLdb.Error as e:
cursor_class=mysql_driver.cursors.DictCursor)
except mysql_driver.Error as e:
module.fail_json(
msg="unable to connect to ProxySQL Admin Module.. %s" % to_native(e)
)
@ -390,7 +380,7 @@ def main():
" need to be updated.")
result['schedules'] = \
proxysql_schedule.get_schedule_config(cursor)
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to modify schedule.. %s" % to_native(e)
)
@ -413,7 +403,7 @@ def main():
result['changed'] = False
result['msg'] = ("The schedule is already absent from the" +
" memory configuration")
except MySQLdb.Error as e:
except mysql_driver.Error as e:
module.fail_json(
msg="unable to remove schedule.. %s" % to_native(e)
)

12
lib/ansible/utils/module_docs_fragments/mysql.py
View file

@ -52,7 +52,8 @@ options:
ssl_ca:
version_added: "2.0"
description:
- The path to a Certificate Authority (CA) certificate. This option, if used, must specify the same certificate as used by the server.
- The path to a Certificate Authority (CA) certificate. This option, if used, must specify the same certificate
as used by the server.
ssl_cert:
version_added: "2.0"
description:
@ -62,11 +63,12 @@ options:
description:
- The path to the client private key.
requirements:
- MySQLdb
- PyMySQL (Python 2.7 and Python 3.X), or
- MySQLdb (Python 2.x)
notes:
- Requires the MySQLdb Python package on the remote host. For Ubuntu, this
is as easy as apt-get install python-mysqldb. (See M(apt).) For CentOS/Fedora, this
is as easy as yum install MySQL-python. (See M(yum).)
- Requires the PyMySQL (Python 2.7 and Python 3.X) or MySQL-python (Python 2.X) Python package on the remote host.
For Ubuntu, this is as easy as apt-get install python-pymysql. (See M(apt).) For CentOS/Fedora, this
is as easy as yum install python2-PyMySQL. (See M(yum).)
- Both C(login_password) and C(login_user) are required when you are
passing credentials. If none are present, the module will attempt to read
the credentials from C(~/.my.cnf), and finally fall back to using the MySQL

3
lib/ansible/utils/module_docs_fragments/proxysql.py
View file

@ -27,7 +27,8 @@ options:
are to be read.
default: ''
requirements:
- MySQLdb
- PyMySQL (Python 2.7 and Python 3.X), or
- MySQLdb (Python 2.x)
'''
# Documentation fragment for managing ProxySQL configuration

2
test/integration/targets/setup_mysql_db/vars/Fedora-py3.yml
View file

@ -2,5 +2,5 @@ mysql_service: 'mariadb'
mysql_packages:
- mariadb-server
- python3-mysql
- python3-PyMySQL
- bzip2

2
test/integration/targets/setup_mysql_db/vars/Fedora.yml
View file

@ -2,5 +2,5 @@ mysql_service: 'mariadb'
mysql_packages:
- mariadb-server
- MySQL-python
- python-PyMySQL
- bzip2

2
test/integration/targets/setup_mysql_db/vars/FreeBSD.yml
View file

@ -2,4 +2,4 @@ mysql_service: 'mysql-server'
mysql_packages:
- mariadb101-server
- py27-mysql-connector-python2
- py-pymysql

2
test/integration/targets/setup_mysql_db/vars/Suse.yml
View file

@ -2,5 +2,5 @@ mysql_service: 'mysql'
mysql_packages:
- mariadb
- python-MySQL-python
- python-PyMySQL
- bzip2

2
test/integration/targets/setup_mysql_db/vars/Ubuntu-16-py3.yml
View file

@ -2,5 +2,5 @@ mysql_service: 'mysql'
mysql_packages:
- mysql-server
- python3-mysqldb
- python3-pymysql
- bzip2