mysql_query: new module (#65941)

* tmp

* mysql_query: new module

* remove irrelevant changes from mysql.py

* add autocommit keyarg to mysql_connect

* add basic CI tests

* tmp

* fix CI tests

* add CI tests

* fix comment in CI tests

* remove path_to_script option

* refactoring

* change tests formatting

* add single_transaction param

* fix sanity

* add note

* fix
This commit is contained in:
Andrew Klychkov 2020-01-09 09:39:09 +03:00 committed by Felix Fontein
parent 963b8249ec
commit 70bc351f31
19 changed files with 566 additions and 52 deletions

View file

@ -44,7 +44,7 @@ mysql_driver_fail_msg = 'The PyMySQL (Python 2.7 and Python 3.X) or MySQL-python
def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None,
connect_timeout=30):
connect_timeout=30, autocommit=False):
config = {}
if ssl_ca is not None or ssl_key is not None or ssl_cert is not None:
@ -76,12 +76,19 @@ def mysql_connect(module, login_user=None, login_password=None, config_file='',
if connect_timeout is not None:
config['connect_timeout'] = connect_timeout
if _mysql_cursor_param == 'cursor':
# In case of PyMySQL driver:
db_connection = mysql_driver.connect(autocommit=autocommit, **config)
else:
# In case of MySQLdb driver
db_connection = mysql_driver.connect(**config)
if autocommit:
db_connection.autocommit(True)
if cursor_class == 'DictCursor':
return db_connection.cursor(**{_mysql_cursor_param: mysql_driver.cursors.DictCursor})
return db_connection.cursor(**{_mysql_cursor_param: mysql_driver.cursors.DictCursor}), db_connection
else:
return db_connection.cursor()
return db_connection.cursor(), db_connection
def mysql_common_argument_spec():

View file

@ -501,7 +501,7 @@ def main():
if db == ['all']:
module.fail_json(msg="name is not allowed to equal 'all' unless state equals import, or dump.")
try:
cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca,
cursor, db_conn = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca,
connect_timeout=connect_timeout)
except Exception as e:
if os.path.exists(config_file):

View file

@ -513,7 +513,7 @@ def main():
module.fail_json(msg=mysql_driver_fail_msg)
try:
cursor = mysql_connect(module, login_user, login_password,
cursor, db_conn = mysql_connect(module, login_user, login_password,
config_file, ssl_cert, ssl_key, ssl_ca, db,
connect_timeout=connect_timeout, cursor_class='DictCursor')
except Exception as e:

View file

@ -0,0 +1,237 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright: (c) 2020, Andrew Klychkov (@Andersson007) <aaklychkov@mail.ru>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
ANSIBLE_METADATA = {
'metadata_version': '1.1',
'supported_by': 'community',
'status': ['preview']
}
DOCUMENTATION = r'''
---
module: mysql_query
short_description: Run MySQL queries
description:
- Runs arbitrary MySQL queries.
- Pay attention, the module does not support check mode!
All queries will be executed in autocommit mode.
version_added: '2.10'
options:
query:
description:
- SQL query to run. Multiple queries can be passed using YAML list syntax.
type: list
elements: str
required: yes
positional_args:
description:
- List of values to be passed as positional arguments to the query.
- Mutually exclusive with I(named_args).
type: list
named_args:
description:
- Dictionary of key-value arguments to pass to the query.
- Mutually exclusive with I(positional_args).
type: dict
login_db:
description:
- Name of database to connect to and run queries against.
type: str
single_transaction:
description:
- Where passed queries run in a single transaction (C(yes)) or commit them one-by-one (C(no)).
type: bool
default: no
notes:
- To pass a query containing commas, use YAML list notation with hyphen (see EXAMPLES block).
author:
- Andrew Klychkov (@Andersson007)
extends_documentation_fragment: mysql
'''
EXAMPLES = r'''
- name: Simple select query to acme db
mysql_query:
login_db: acme
query: SELECT * FROM orders
- name: Select query to db acme with positional arguments
mysql_query:
login_db: acme
query: SELECT * FROM acme WHERE id = %s AND story = %s
positional_args:
- 1
- test
- name: Select query to test_db with named_args
mysql_query:
login_db: test_db
query: SELECT * FROM test WHERE id = %(id_val)s AND story = %(story_val)s
named_args:
id_val: 1
story_val: test
- name: Run several insert queries against db test_db in single transaction
mysql_query:
login_db: test_db
query:
- INSERT INTO articles (id, story) VALUES (2, 'my_long_story')
- INSERT INTO prices (id, price) VALUES (123, '100.00')
single_transaction: yes
'''
RETURN = r'''
executed_queries:
description: List of executed queries.
returned: always
type: list
sample: ['SELECT * FROM bar', 'UPDATE bar SET id = 1 WHERE id = 2']
query_result:
description:
- List of lists (sublist for each query) containing dictionaries
in column:value form representing returned rows.
returned: changed
type: list
sample: [[{"Column": "Value1"},{"Column": "Value2"}], [{"ID": 1}, {"ID": 2}]]
rowcount:
description: Number of affected rows for each subquery.
returned: changed
type: list
sample: [5, 1]
'''
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.mysql import (
mysql_connect,
mysql_common_argument_spec,
mysql_driver,
mysql_driver_fail_msg,
)
from ansible.module_utils._text import to_native
DML_QUERY_KEYWORDS = ('INSERT', 'UPDATE', 'DELETE')
# TRUNCATE is not DDL query but it also returns 0 rows affected:
DDL_QUERY_KEYWORDS = ('CREATE', 'DROP', 'ALTER', 'RENAME', 'TRUNCATE')
# ===========================================
# Module execution.
#
def main():
argument_spec = mysql_common_argument_spec()
argument_spec.update(
query=dict(type='list', elements='str', required=True),
login_db=dict(type='str'),
positional_args=dict(type='list'),
named_args=dict(type='dict'),
single_transaction=dict(type='bool', default=False),
)
module = AnsibleModule(
argument_spec=argument_spec,
mutually_exclusive=(
('positional_args', 'named_args'),
),
)
db = module.params['login_db']
connect_timeout = module.params['connect_timeout']
login_user = module.params['login_user']
login_password = module.params['login_password']
ssl_cert = module.params['client_cert']
ssl_key = module.params['client_key']
ssl_ca = module.params['ca_cert']
config_file = module.params['config_file']
query = module.params["query"]
if module.params["single_transaction"]:
autocommit = False
else:
autocommit = True
# Prepare args:
if module.params.get("positional_args"):
arguments = module.params["positional_args"]
elif module.params.get("named_args"):
arguments = module.params["named_args"]
else:
arguments = None
if mysql_driver is None:
module.fail_json(msg=mysql_driver_fail_msg)
# Connect to DB:
try:
cursor, db_connection = mysql_connect(module, login_user, login_password,
config_file, ssl_cert, ssl_key, ssl_ca, db,
connect_timeout=connect_timeout,
cursor_class='DictCursor', autocommit=autocommit)
except Exception as e:
module.fail_json(msg="unable to connect to database, check login_user and "
"login_password are correct or %s has the credentials. "
"Exception message: %s" % (config_file, to_native(e)))
# Set defaults:
changed = False
max_keyword_len = len(max(DML_QUERY_KEYWORDS + DDL_QUERY_KEYWORDS, key=len))
# Execute query:
query_result = []
executed_queries = []
rowcount = []
for q in query:
try:
cursor.execute(q, arguments)
except Exception as e:
if not autocommit:
db_connection.rollback()
cursor.close()
module.fail_json(msg="Cannot execute SQL '%s' args [%s]: %s" % (q, arguments, to_native(e)))
try:
query_result.append([dict(row) for row in cursor.fetchall()])
except Exception as e:
if not autocommit:
db_connection.rollback()
module.fail_json(msg="Cannot fetch rows from cursor: %s" % to_native(e))
# Check DML or DDL keywords in query and set changed accordingly:
q = q.lstrip()[0:max_keyword_len].upper()
for keyword in DML_QUERY_KEYWORDS:
if keyword in q and cursor.rowcount > 0:
changed = True
for keyword in DDL_QUERY_KEYWORDS:
if keyword in q:
changed = True
executed_queries.append(cursor._last_executed)
rowcount.append(cursor.rowcount)
# When the module run with the single_transaction == True:
if not autocommit:
db_connection.commit()
# Create dict with returned values:
kw = {
'changed': changed,
'executed_queries': executed_queries,
'query_result': query_result,
'rowcount': rowcount,
}
# Exit:
module.exit_json(**kw)
if __name__ == '__main__':
main()

View file

@ -428,7 +428,7 @@ def main():
login_user = module.params["login_user"]
try:
cursor = mysql_connect(module, login_user, login_password, config_file,
cursor, db_conn = mysql_connect(module, login_user, login_password, config_file,
ssl_cert, ssl_key, ssl_ca, None, cursor_class='DictCursor',
connect_timeout=connect_timeout)
except Exception as e:

View file

@ -714,13 +714,13 @@ def main():
try:
if check_implicit_admin:
try:
cursor = mysql_connect(module, 'root', '', config_file, ssl_cert, ssl_key, ssl_ca, db,
cursor, db_conn = mysql_connect(module, 'root', '', config_file, ssl_cert, ssl_key, ssl_ca, db,
connect_timeout=connect_timeout)
except Exception:
pass
if not cursor:
cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, db,
cursor, db_conn = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, db,
connect_timeout=connect_timeout)
except Exception as e:
module.fail_json(msg="unable to connect to database, check login_user and login_password are correct or %s has the credentials. "

View file

@ -212,7 +212,7 @@ def main():
warnings.filterwarnings('error', category=mysql_driver.Warning)
try:
cursor = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,
cursor, db_conn = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,
connect_timeout=connect_timeout)
except Exception as e:
if os.path.exists(config_file):

View file

@ -444,7 +444,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file,

View file

@ -197,7 +197,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file,

View file

@ -188,7 +188,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file)

View file

@ -416,7 +416,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file,

View file

@ -543,7 +543,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file,

View file

@ -314,7 +314,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file,

View file

@ -354,7 +354,7 @@ def main():
cursor = None
try:
cursor = mysql_connect(module,
cursor, db_conn = mysql_connect(module,
login_user,
login_password,
config_file,

View file

@ -0,0 +1,8 @@
destructive
shippable/posix/group3
skip/osx
skip/freebsd
skip/ubuntu
skip/fedora
skip/opensuse
skip/rhel

View file

@ -0,0 +1,6 @@
root_user: root
db_name: data
test_db: testdb
test_table1: test1
test_table2: test2
test_script_path: /tmp/test.sql

View file

@ -0,0 +1,2 @@
dependencies:
- setup_mysql8

View file

@ -0,0 +1,5 @@
# mysql_query module initial CI tests
- import_tasks: mysql_query_initial.yml
when:
- ansible_distribution == 'CentOS'
- ansible_distribution_major_version >= '7'

View file

@ -0,0 +1,249 @@
# Test code for mysql_query module
# Copyright: (c) 2020, Andrew Klychkov (@Andersson007) <aaklychkov@mail.ru>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
- vars:
mysql_parameters: &mysql_params
login_unix_socket: '{{ mysql_socket }}'
login_user: '{{ root_user }}'
login_password: '{{ root_pass }}'
block:
- name: Create db {{ test_db }}
mysql_query:
<<: *mysql_params
query: 'CREATE DATABASE {{ test_db }}'
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['CREATE DATABASE {{ test_db }}']
- name: Create {{ test_table1 }}
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'CREATE TABLE {{ test_table1 }} (id int)'
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['CREATE TABLE {{ test_table1 }} (id int)']
- name: Insert test data
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query:
- 'INSERT INTO {{ test_table1 }} VALUES (1), (2)'
- 'INSERT INTO {{ test_table1 }} VALUES (3)'
single_transaction: yes
register: result
- assert:
that:
- result is changed
- result.rowcount == [2, 1]
- result.executed_queries == ['INSERT INTO {{ test_table1 }} VALUES (1), (2)', 'INSERT INTO {{ test_table1 }} VALUES (3)']
- name: Check data in {{ test_table1 }}
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table1 }}'
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ['SELECT * FROM {{ test_table1 }}']
- result.rowcount == [3]
- result.query_result[0][0].id == 1
- result.query_result[0][1].id == 2
- result.query_result[0][2].id == 3
- name: Check data in {{ test_table1 }} using positional args
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table1 }} WHERE id = %s'
positional_args:
- 1
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ["SELECT * FROM {{ test_table1 }} WHERE id = 1"]
- result.rowcount == [1]
- result.query_result[0][0].id == 1
- name: Check data in {{ test_table1 }} using named args
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table1 }} WHERE id = %(some_id)s'
named_args:
some_id: 1
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ["SELECT * FROM {{ test_table1 }} WHERE id = 1"]
- result.rowcount == [1]
- result.query_result[0][0].id == 1
- name: Update data in {{ test_table1 }}
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'UPDATE {{ test_table1 }} SET id = %(new_id)s WHERE id = %(current_id)s'
named_args:
current_id: 1
new_id: 0
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['UPDATE {{ test_table1 }} SET id = 0 WHERE id = 1']
- result.rowcount == [1]
- name: Check the prev update - row with value 1 does not exist anymore
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table1 }} WHERE id = %(some_id)s'
named_args:
some_id: 1
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ['SELECT * FROM {{ test_table1 }} WHERE id = 1']
- result.rowcount == [0]
- name: Check the prev update - row with value - exist
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table1 }} WHERE id = %(some_id)s'
named_args:
some_id: 0
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ['SELECT * FROM {{ test_table1 }} WHERE id = 0']
- result.rowcount == [1]
- name: Update data in {{ test_table1 }} again
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'UPDATE {{ test_table1 }} SET id = %(new_id)s WHERE id = %(current_id)s'
named_args:
current_id: 1
new_id: 0
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ['UPDATE {{ test_table1 }} SET id = 0 WHERE id = 1']
- result.rowcount == [0]
- name: Delete data from {{ test_table1 }}
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query:
- 'DELETE FROM {{ test_table1 }} WHERE id = 0'
- 'SELECT * FROM {{ test_table1 }} WHERE id = 0'
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['DELETE FROM {{ test_table1 }} WHERE id = 0', 'SELECT * FROM {{ test_table1 }} WHERE id = 0']
- result.rowcount == [1, 0]
- name: Delete data from {{ test_table1 }} again
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'DELETE FROM {{ test_table1 }} WHERE id = 0'
register: result
- assert:
that:
- result is not changed
- result.executed_queries == ['DELETE FROM {{ test_table1 }} WHERE id = 0']
- result.rowcount == [0]
- name: Truncate {{ test_table1 }}
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query:
- 'TRUNCATE {{ test_table1 }}'
- 'SELECT * FROM {{ test_table1 }}'
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['TRUNCATE {{ test_table1 }}', 'SELECT * FROM {{ test_table1 }}']
- result.rowcount == [0, 0]
- name: Rename {{ test_table1 }}
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'RENAME TABLE {{ test_table1 }} TO {{ test_table2 }}'
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['RENAME TABLE {{ test_table1 }} TO {{ test_table2 }}']
- result.rowcount == [0]
- name: Check the prev rename
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table1 }}'
register: result
ignore_errors: yes
- assert:
that:
- result.failed == true
- name: Check the prev rename
mysql_query:
<<: *mysql_params
login_db: '{{ test_db }}'
query: 'SELECT * FROM {{ test_table2 }}'
register: result
- assert:
that:
- result.rowcount == [0]
- name: Drop db {{ test_db }}
mysql_query:
<<: *mysql_params
query: 'DROP DATABASE {{ test_db }}'
register: result
- assert:
that:
- result is changed
- result.executed_queries == ['DROP DATABASE {{ test_db }}']