mysql_query: new module (#65941)

* tmp

* mysql_query: new module

* remove irrelevant changes from mysql.py

* add autocommit keyarg to mysql_connect

* add basic CI tests

* tmp

* fix CI tests

* add CI tests

* fix comment in CI tests

* remove path_to_script option

* refactoring

* change tests formatting

* add single_transaction param

* fix sanity

* add note

* fix

lib/ansible/module_utils/mysql.py

@@ -44,7 +44,7 @@ mysql_driver_fail_msg = 'The PyMySQL (Python 2.7 and Python 3.X) or MySQL-python
 
 
 def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None,
-                  connect_timeout=30):
+                  connect_timeout=30, autocommit=False):
     config = {}
 
     if ssl_ca is not None or ssl_key is not None or ssl_cert is not None:
@@ -76,12 +76,19 @@ def mysql_connect(module, login_user=None, login_password=None, config_file='',
     if connect_timeout is not None:
         config['connect_timeout'] = connect_timeout
 
+    if _mysql_cursor_param == 'cursor':
+        # In case of PyMySQL driver:
+        db_connection = mysql_driver.connect(autocommit=autocommit, **config)
+    else:
+        # In case of MySQLdb driver
         db_connection = mysql_driver.connect(**config)
+        if autocommit:
+            db_connection.autocommit(True)
 
     if cursor_class == 'DictCursor':
-        return db_connection.cursor(**{_mysql_cursor_param: mysql_driver.cursors.DictCursor})
+        return db_connection.cursor(**{_mysql_cursor_param: mysql_driver.cursors.DictCursor}), db_connection
     else:
-        return db_connection.cursor()
+        return db_connection.cursor(), db_connection
 
 
 def mysql_common_argument_spec():

lib/ansible/modules/database/mysql/mysql_db.py

@@ -501,7 +501,7 @@ def main():
         if db == ['all']:
             module.fail_json(msg="name is not allowed to equal 'all' unless state equals import, or dump.")
     try:
-        cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca,
+        cursor, db_conn = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca,
                                         connect_timeout=connect_timeout)
     except Exception as e:
         if os.path.exists(config_file):

lib/ansible/modules/database/mysql/mysql_info.py

@@ -513,7 +513,7 @@ def main():
         module.fail_json(msg=mysql_driver_fail_msg)
 
     try:
-        cursor = mysql_connect(module, login_user, login_password,
+        cursor, db_conn = mysql_connect(module, login_user, login_password,
                                         config_file, ssl_cert, ssl_key, ssl_ca, db,
                                         connect_timeout=connect_timeout, cursor_class='DictCursor')
     except Exception as e:

lib/ansible/modules/database/mysql/mysql_query.py

@@ -0,0 +1,237 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2020, Andrew Klychkov (@Andersson007) <aaklychkov@mail.ru>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'supported_by': 'community',
+    'status': ['preview']
+}
+
+DOCUMENTATION = r'''
+---
+module: mysql_query
+short_description: Run MySQL queries
+description:
+- Runs arbitrary MySQL queries.
+- Pay attention, the module does not support check mode!
+  All queries will be executed in autocommit mode.
+version_added: '2.10'
+options:
+  query:
+    description:
+    - SQL query to run. Multiple queries can be passed using YAML list syntax.
+    type: list
+    elements: str
+    required: yes
+  positional_args:
+    description:
+    - List of values to be passed as positional arguments to the query.
+    - Mutually exclusive with I(named_args).
+    type: list
+  named_args:
+    description:
+    - Dictionary of key-value arguments to pass to the query.
+    - Mutually exclusive with I(positional_args).
+    type: dict
+  login_db:
+    description:
+    - Name of database to connect to and run queries against.
+    type: str
+  single_transaction:
+    description:
+    - Where passed queries run in a single transaction (C(yes)) or commit them one-by-one (C(no)).
+    type: bool
+    default: no
+notes:
+- To pass a query containing commas, use YAML list notation with hyphen (see EXAMPLES block).
+author:
+- Andrew Klychkov (@Andersson007)
+extends_documentation_fragment: mysql
+'''
+
+EXAMPLES = r'''
+- name: Simple select query to acme db
+  mysql_query:
+    login_db: acme
+    query: SELECT * FROM orders
+
+- name: Select query to db acme with positional arguments
+  mysql_query:
+    login_db: acme
+    query: SELECT * FROM acme WHERE id = %s AND story = %s
+    positional_args:
+    - 1
+    - test
+
+- name: Select query to test_db with named_args
+  mysql_query:
+    login_db: test_db
+    query: SELECT * FROM test WHERE id = %(id_val)s AND story = %(story_val)s
+    named_args:
+      id_val: 1
+      story_val: test
+
+- name: Run several insert queries against db test_db in single transaction
+  mysql_query:
+    login_db: test_db
+    query:
+    - INSERT INTO articles (id, story) VALUES (2, 'my_long_story')
+    - INSERT INTO prices (id, price) VALUES (123, '100.00')
+    single_transaction: yes
+'''
+
+RETURN = r'''
+executed_queries:
+    description: List of executed queries.
+    returned: always
+    type: list
+    sample: ['SELECT * FROM bar', 'UPDATE bar SET id = 1 WHERE id = 2']
+query_result:
+    description:
+    - List of lists (sublist for each query) containing dictionaries
+      in column:value form representing returned rows.
+    returned: changed
+    type: list
+    sample: [[{"Column": "Value1"},{"Column": "Value2"}], [{"ID": 1}, {"ID": 2}]]
+rowcount:
+    description: Number of affected rows for each subquery.
+    returned: changed
+    type: list
+    sample: [5, 1]
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.mysql import (
+    mysql_connect,
+    mysql_common_argument_spec,
+    mysql_driver,
+    mysql_driver_fail_msg,
+)
+from ansible.module_utils._text import to_native
+
+DML_QUERY_KEYWORDS = ('INSERT', 'UPDATE', 'DELETE')
+# TRUNCATE is not DDL query but it also returns 0 rows affected:
+DDL_QUERY_KEYWORDS = ('CREATE', 'DROP', 'ALTER', 'RENAME', 'TRUNCATE')
+
+
+# ===========================================
+# Module execution.
+#
+
+def main():
+    argument_spec = mysql_common_argument_spec()
+    argument_spec.update(
+        query=dict(type='list', elements='str', required=True),
+        login_db=dict(type='str'),
+        positional_args=dict(type='list'),
+        named_args=dict(type='dict'),
+        single_transaction=dict(type='bool', default=False),
+    )
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        mutually_exclusive=(
+            ('positional_args', 'named_args'),
+        ),
+    )
+
+    db = module.params['login_db']
+    connect_timeout = module.params['connect_timeout']
+    login_user = module.params['login_user']
+    login_password = module.params['login_password']
+    ssl_cert = module.params['client_cert']
+    ssl_key = module.params['client_key']
+    ssl_ca = module.params['ca_cert']
+    config_file = module.params['config_file']
+    query = module.params["query"]
+    if module.params["single_transaction"]:
+        autocommit = False
+    else:
+        autocommit = True
+    # Prepare args:
+    if module.params.get("positional_args"):
+        arguments = module.params["positional_args"]
+    elif module.params.get("named_args"):
+        arguments = module.params["named_args"]
+    else:
+        arguments = None
+
+    if mysql_driver is None:
+        module.fail_json(msg=mysql_driver_fail_msg)
+
+    # Connect to DB:
+    try:
+        cursor, db_connection = mysql_connect(module, login_user, login_password,
+                                              config_file, ssl_cert, ssl_key, ssl_ca, db,
+                                              connect_timeout=connect_timeout,
+                                              cursor_class='DictCursor', autocommit=autocommit)
+    except Exception as e:
+        module.fail_json(msg="unable to connect to database, check login_user and "
+                             "login_password are correct or %s has the credentials. "
+                             "Exception message: %s" % (config_file, to_native(e)))
+    # Set defaults:
+    changed = False
+
+    max_keyword_len = len(max(DML_QUERY_KEYWORDS + DDL_QUERY_KEYWORDS, key=len))
+
+    # Execute query:
+    query_result = []
+    executed_queries = []
+    rowcount = []
+    for q in query:
+        try:
+            cursor.execute(q, arguments)
+
+        except Exception as e:
+            if not autocommit:
+                db_connection.rollback()
+
+            cursor.close()
+            module.fail_json(msg="Cannot execute SQL '%s' args [%s]: %s" % (q, arguments, to_native(e)))
+
+        try:
+            query_result.append([dict(row) for row in cursor.fetchall()])
+
+        except Exception as e:
+            if not autocommit:
+                db_connection.rollback()
+
+            module.fail_json(msg="Cannot fetch rows from cursor: %s" % to_native(e))
+
+        # Check DML or DDL keywords in query and set changed accordingly:
+        q = q.lstrip()[0:max_keyword_len].upper()
+        for keyword in DML_QUERY_KEYWORDS:
+            if keyword in q and cursor.rowcount > 0:
+                changed = True
+
+        for keyword in DDL_QUERY_KEYWORDS:
+            if keyword in q:
+                changed = True
+
+        executed_queries.append(cursor._last_executed)
+        rowcount.append(cursor.rowcount)
+
+    # When the module run with the single_transaction == True:
+    if not autocommit:
+        db_connection.commit()
+
+    # Create dict with returned values:
+    kw = {
+        'changed': changed,
+        'executed_queries': executed_queries,
+        'query_result': query_result,
+        'rowcount': rowcount,
+    }
+
+    # Exit:
+    module.exit_json(**kw)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/modules/database/mysql/mysql_replication.py

@@ -428,7 +428,7 @@ def main():
     login_user = module.params["login_user"]
 
     try:
-        cursor = mysql_connect(module, login_user, login_password, config_file,
+        cursor, db_conn = mysql_connect(module, login_user, login_password, config_file,
                                         ssl_cert, ssl_key, ssl_ca, None, cursor_class='DictCursor',
                                         connect_timeout=connect_timeout)
     except Exception as e:

lib/ansible/modules/database/mysql/mysql_user.py

@@ -714,13 +714,13 @@ def main():
     try:
         if check_implicit_admin:
             try:
-                cursor = mysql_connect(module, 'root', '', config_file, ssl_cert, ssl_key, ssl_ca, db,
+                cursor, db_conn = mysql_connect(module, 'root', '', config_file, ssl_cert, ssl_key, ssl_ca, db,
                                                 connect_timeout=connect_timeout)
             except Exception:
                 pass
 
         if not cursor:
-            cursor = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, db,
+            cursor, db_conn = mysql_connect(module, login_user, login_password, config_file, ssl_cert, ssl_key, ssl_ca, db,
                                             connect_timeout=connect_timeout)
     except Exception as e:
         module.fail_json(msg="unable to connect to database, check login_user and login_password are correct or %s has the credentials. "

lib/ansible/modules/database/mysql/mysql_variables.py

@@ -212,7 +212,7 @@ def main():
         warnings.filterwarnings('error', category=mysql_driver.Warning)
 
     try:
-        cursor = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,
+        cursor, db_conn = mysql_connect(module, user, password, config_file, ssl_cert, ssl_key, ssl_ca, db,
                                         connect_timeout=connect_timeout)
     except Exception as e:
         if os.path.exists(config_file):

lib/ansible/modules/database/proxysql/proxysql_backend_servers.py

@@ -444,7 +444,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file,

lib/ansible/modules/database/proxysql/proxysql_global_variables.py

@@ -197,7 +197,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file,

lib/ansible/modules/database/proxysql/proxysql_manage_config.py

@@ -188,7 +188,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file)

lib/ansible/modules/database/proxysql/proxysql_mysql_users.py

@@ -416,7 +416,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file,

lib/ansible/modules/database/proxysql/proxysql_query_rules.py

@@ -543,7 +543,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file,

lib/ansible/modules/database/proxysql/proxysql_replication_hostgroups.py

@@ -314,7 +314,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file,

lib/ansible/modules/database/proxysql/proxysql_scheduler.py

@@ -354,7 +354,7 @@ def main():
 
     cursor = None
     try:
-        cursor = mysql_connect(module,
+        cursor, db_conn = mysql_connect(module,
                                         login_user,
                                         login_password,
                                         config_file,

test/integration/targets/mysql_query/aliases

@@ -0,0 +1,8 @@
+destructive
+shippable/posix/group3
+skip/osx
+skip/freebsd
+skip/ubuntu
+skip/fedora
+skip/opensuse
+skip/rhel

test/integration/targets/mysql_query/defaults/main.yml

@@ -0,0 +1,6 @@
+root_user: root
+db_name: data
+test_db: testdb
+test_table1: test1
+test_table2: test2
+test_script_path: /tmp/test.sql

test/integration/targets/mysql_query/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+- setup_mysql8

test/integration/targets/mysql_query/tasks/main.yml

@@ -0,0 +1,5 @@
+# mysql_query module initial CI tests
+- import_tasks: mysql_query_initial.yml
+  when:
+  - ansible_distribution == 'CentOS'
+  - ansible_distribution_major_version >= '7'

test/integration/targets/mysql_query/tasks/mysql_query_initial.yml

@@ -0,0 +1,249 @@
+# Test code for mysql_query module
+# Copyright: (c) 2020, Andrew Klychkov (@Andersson007) <aaklychkov@mail.ru>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+- vars:
+    mysql_parameters: &mysql_params
+      login_unix_socket: '{{ mysql_socket }}'
+      login_user: '{{ root_user }}'
+      login_password: '{{ root_pass }}'
+
+  block:
+
+  - name: Create db {{ test_db }}
+    mysql_query:
+      <<: *mysql_params
+      query: 'CREATE DATABASE {{ test_db }}'
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['CREATE DATABASE {{ test_db }}']
+
+  - name: Create {{ test_table1 }}
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'CREATE TABLE {{ test_table1 }} (id int)'
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['CREATE TABLE {{ test_table1 }} (id int)']
+
+  - name: Insert test data
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query:
+      - 'INSERT INTO {{ test_table1 }} VALUES (1), (2)'
+      - 'INSERT INTO {{ test_table1 }} VALUES (3)'
+      single_transaction: yes
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.rowcount == [2, 1]
+      - result.executed_queries == ['INSERT INTO {{ test_table1 }} VALUES (1), (2)', 'INSERT INTO {{ test_table1 }} VALUES (3)']
+
+  - name: Check data in {{ test_table1 }}
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table1 }}'
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ['SELECT * FROM {{ test_table1 }}']
+      - result.rowcount == [3]
+      - result.query_result[0][0].id == 1
+      - result.query_result[0][1].id == 2
+      - result.query_result[0][2].id == 3
+
+  - name: Check data in {{ test_table1 }} using positional args
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table1 }} WHERE id = %s'
+      positional_args:
+      - 1
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ["SELECT * FROM {{ test_table1 }} WHERE id = 1"]
+      - result.rowcount == [1]
+      - result.query_result[0][0].id == 1
+
+  - name: Check data in {{ test_table1 }} using named args
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table1 }} WHERE id = %(some_id)s'
+      named_args:
+        some_id: 1
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ["SELECT * FROM {{ test_table1 }} WHERE id = 1"]
+      - result.rowcount == [1]
+      - result.query_result[0][0].id == 1
+
+  - name: Update data in {{ test_table1 }}
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'UPDATE {{ test_table1 }} SET id = %(new_id)s WHERE id = %(current_id)s'
+      named_args:
+        current_id: 1
+        new_id: 0
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['UPDATE {{ test_table1 }} SET id = 0 WHERE id = 1']
+      - result.rowcount == [1]
+
+  - name: Check the prev update - row with value 1 does not exist anymore
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table1 }} WHERE id = %(some_id)s'
+      named_args:
+        some_id: 1
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ['SELECT * FROM {{ test_table1 }} WHERE id = 1']
+      - result.rowcount == [0]
+
+  - name: Check the prev update - row with value - exist
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table1 }} WHERE id = %(some_id)s'
+      named_args:
+        some_id: 0
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ['SELECT * FROM {{ test_table1 }} WHERE id = 0']
+      - result.rowcount == [1]
+
+  - name: Update data in {{ test_table1 }} again
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'UPDATE {{ test_table1 }} SET id = %(new_id)s WHERE id = %(current_id)s'
+      named_args:
+        current_id: 1
+        new_id: 0
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ['UPDATE {{ test_table1 }} SET id = 0 WHERE id = 1']
+      - result.rowcount == [0]
+
+  - name: Delete data from {{ test_table1 }}
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query:
+      - 'DELETE FROM {{ test_table1 }} WHERE id = 0'
+      - 'SELECT * FROM {{ test_table1 }} WHERE id = 0'
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['DELETE FROM {{ test_table1 }} WHERE id = 0', 'SELECT * FROM {{ test_table1 }} WHERE id = 0']
+      - result.rowcount == [1, 0]
+
+  - name: Delete data from {{ test_table1 }} again
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'DELETE FROM {{ test_table1 }} WHERE id = 0'
+    register: result
+
+  - assert:
+      that:
+      - result is not changed
+      - result.executed_queries == ['DELETE FROM {{ test_table1 }} WHERE id = 0']
+      - result.rowcount == [0]
+
+  - name: Truncate {{ test_table1 }}
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query:
+      - 'TRUNCATE {{ test_table1 }}'
+      - 'SELECT * FROM {{ test_table1 }}'
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['TRUNCATE {{ test_table1 }}', 'SELECT * FROM {{ test_table1 }}']
+      - result.rowcount == [0, 0]
+
+  - name: Rename {{ test_table1 }}
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'RENAME TABLE {{ test_table1 }} TO {{ test_table2 }}'
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['RENAME TABLE {{ test_table1 }} TO {{ test_table2 }}']
+      - result.rowcount == [0]
+
+  - name: Check the prev rename
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table1 }}'
+    register: result
+    ignore_errors: yes
+
+  - assert:
+      that:
+      - result.failed == true
+
+  - name: Check the prev rename
+    mysql_query:
+      <<: *mysql_params
+      login_db: '{{ test_db }}'
+      query: 'SELECT * FROM {{ test_table2 }}'
+    register: result
+
+  - assert:
+      that:
+      - result.rowcount == [0]
+
+  - name: Drop db {{ test_db }}
+    mysql_query:
+      <<: *mysql_params
+      query: 'DROP DATABASE {{ test_db }}'
+    register: result
+
+  - assert:
+      that:
+      - result is changed
+      - result.executed_queries == ['DROP DATABASE {{ test_db }}']