Add Rest API to DNS (#59937)

* add rest to dns

* fix issues with tests

* fixes to netapp.py

* fixes

* fixes

* add updates

* fix issues

lib/ansible/module_utils/netapp.py

@@ -134,6 +134,16 @@ def ontap_sf_host_argument_spec():
     )
 
 
+def aws_cvs_host_argument_spec():
+
+    return dict(
+        api_url=dict(required=True, type='str'),
+        validate_certs=dict(required=False, type='bool', default=True),
+        api_key=dict(required=True, type='str'),
+        secret_key=dict(required=True, type='str')
+    )
+
+
 def create_sf_connection(module, port=None):
     hostname = module.params['hostname']
     username = module.params['username']
@@ -575,7 +585,7 @@ class OntapRestAPI(object):
             response.raise_for_status()
             json_dict, json_error = get_json(response)
         except requests.exceptions.HTTPError as err:
-            junk, json_error = get_json(response)
+            __, json_error = get_json(response)
             if json_error is None:
                 self.log_error(status_code, 'HTTP error: %s' % err)
                 error_details = str(err)
@@ -610,17 +620,30 @@ class OntapRestAPI(object):
         method = 'DELETE'
         return self.send_request(method, api, params, json=data)
 
-    def is_rest(self):
+    def _is_rest(self, used_unsupported_rest_properties=None):
         if self.use_rest == "Always":
-            return True
+            if used_unsupported_rest_properties:
-        if self.use_rest == 'Never':
+                error = "REST API currently does not support '%s'" % \
-            return False
+                        ', '.join(used_unsupported_rest_properties)
+                return True, error
+            else:
+                return True, None
+        if self.use_rest == 'Never' or used_unsupported_rest_properties:
+            # force ZAPI if requested or if some parameter requires it
+            return False, None
         method = 'HEAD'
         api = 'cluster/software'
-        status_code, junk = self.send_request(method, api, params=None, return_status_code=True)
+        status_code, __ = self.send_request(method, api, params=None, return_status_code=True)
         if status_code == 200:
-            return True
+            return True, None
-        return False
+        return False, None
+
+    def is_rest(self, used_unsupported_rest_properties=None):
+        ''' only return error if there is a reason to '''
+        use_rest, error = self._is_rest(used_unsupported_rest_properties)
+        if used_unsupported_rest_properties is None:
+            return use_rest
+        return use_rest, error
 
     def log_error(self, status_code, message):
         self.errors.append(message)
@@ -628,3 +651,94 @@ class OntapRestAPI(object):
 
     def log_debug(self, status_code, content):
         self.debug_logs.append((status_code, content))
+
+
+class AwsCvsRestAPI(object):
+    def __init__(self, module, timeout=60):
+        self.module = module
+        self.api_key = self.module.params['api_key']
+        self.secret_key = self.module.params['secret_key']
+        self.api_url = self.module.params['api_url']
+        self.verify = self.module.params['validate_certs']
+        self.timeout = timeout
+        self.url = 'https://' + self.api_url + '/v1/'
+        self.check_required_library()
+
+    def check_required_library(self):
+        if not HAS_REQUESTS:
+            self.module.fail_json(msg=missing_required_lib('requests'))
+
+    def send_request(self, method, api, params, json=None):
+        ''' send http request and process reponse, including error conditions '''
+        url = self.url + api
+        status_code = None
+        content = None
+        json_dict = None
+        json_error = None
+        error_details = None
+        headers = {
+            'Content-type': "application/json",
+            'api-key': self.api_key,
+            'secret-key': self.secret_key,
+            'Cache-Control': "no-cache",
+        }
+
+        def get_json(response):
+            ''' extract json, and error message if present '''
+            try:
+                json = response.json()
+
+            except ValueError:
+                return None, None
+            success_code = [200, 201, 202]
+            if response.status_code not in success_code:
+                error = json.get('message')
+            else:
+                error = None
+            return json, error
+        try:
+            response = requests.request(method, url, headers=headers, timeout=self.timeout, json=json)
+            status_code = response.status_code
+            # If the response was successful, no Exception will be raised
+            json_dict, json_error = get_json(response)
+        except requests.exceptions.HTTPError as err:
+            __, json_error = get_json(response)
+            if json_error is None:
+                error_details = str(err)
+        except requests.exceptions.ConnectionError as err:
+            error_details = str(err)
+        except Exception as err:
+            error_details = str(err)
+        if json_error is not None:
+            error_details = json_error
+
+        return json_dict, error_details
+
+    # If an error was reported in the json payload, it is handled below
+    def get(self, api, params=None):
+        method = 'GET'
+        return self.send_request(method, api, params)
+
+    def post(self, api, data, params=None):
+        method = 'POST'
+        return self.send_request(method, api, params, json=data)
+
+    def patch(self, api, data, params=None):
+        method = 'PATCH'
+        return self.send_request(method, api, params, json=data)
+
+    def put(self, api, data, params=None):
+        method = 'PUT'
+        return self.send_request(method, api, params, json=data)
+
+    def delete(self, api, data, params=None):
+        method = 'DELETE'
+        return self.send_request(method, api, params, json=data)
+
+    def get_state(self, jobId):
+        """ Method to get the state of the job """
+        method = 'GET'
+        response, status_code = self.get('Jobs/%s' % jobId)
+        while str(response['state']) not in 'done':
+            response, status_code = self.get('Jobs/%s' % jobId)
+        return 'done'

lib/ansible/module_utils/netapp_module.py

@@ -160,6 +160,22 @@ class NetAppModule(object):
             return 'delete'
         return 'create'
 
+    def compare_and_update_values(self, current, desired, keys_to_compare):
+        updated_values = dict()
+        is_changed = False
+        for key in keys_to_compare:
+            if key in current:
+                if key in desired and desired[key] is not None:
+                    if current[key] != desired[key]:
+                        updated_values[key] = desired[key]
+                        is_changed = True
+                    else:
+                        updated_values[key] = current[key]
+                else:
+                    updated_values[key] = current[key]
+
+        return updated_values, is_changed
+
     @staticmethod
     def check_keys(current, desired):
         ''' TODO: raise an error if keys do not match
@@ -170,8 +186,8 @@ class NetAppModule(object):
 
     @staticmethod
     def compare_lists(current, desired, get_list_diff):
-        ''' compares two lists and return a list of elements that are either the desired elements or elements that are modified from the current state
+        ''' compares two lists and return a list of elements that are either the desired elements or elements that are
-        depending on the get_list_diff flag
+            modified from the current state depending on the get_list_diff flag
             :param: current: current item attribute in ONTAP
             :param: desired: attributes from playbook
             :param: get_list_diff: specifies whether to have a diff of desired list w.r.t current list for an attribute

lib/ansible/modules/storage/netapp/na_ontap_dns.py

@@ -70,6 +70,7 @@ from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
 import ansible.module_utils.netapp as netapp_utils
 from ansible.module_utils.netapp_module import NetAppModule
+from ansible.module_utils.netapp import OntapRestAPI
 
 HAS_NETAPP_LIB = netapp_utils.has_netapp_lib()
 
@@ -80,6 +81,7 @@ class NetAppOntapDns(object):
     """
 
     def __init__(self):
+        self.use_rest = False
         self.argument_spec = netapp_utils.na_ontap_host_argument_spec()
         self.argument_spec.update(dict(
             state=dict(required=False, choices=['present', 'absent'], default='present'),
@@ -91,12 +93,24 @@ class NetAppOntapDns(object):
 
         self.module = AnsibleModule(
             argument_spec=self.argument_spec,
+            required_if=[('state', 'present', ['domains', 'nameservers'])],
             supports_check_mode=True
         )
 
         self.na_helper = NetAppModule()
         self.parameters = self.na_helper.set_parameters(self.module.params)
 
+        # REST API should be used for ONTAP 9.6 or higher, ZAPI for lower version
+        self.restApi = OntapRestAPI(self.module)
+        # some attributes are not supported in earlier REST implementation
+        unsupported_rest_properties = ['skip_validation']
+        used_unsupported_rest_properties = [x for x in unsupported_rest_properties if x in self.parameters]
+        self.use_rest, error = self.restApi.is_rest(used_unsupported_rest_properties)
+
+        if error is not None:
+            self.module.fail_json(msg=error)
+
+        if not self.use_rest:
             if HAS_NETAPP_LIB is False:
                 self.module.fail_json(msg="the python NetApp-Lib module is required")
             else:
@@ -108,6 +122,16 @@ class NetAppOntapDns(object):
         Create DNS server
         :return: none
         """
+        if self.use_rest:
+            api = 'name-services/dns'
+            params = {}
+            params['domains'] = self.parameters['domains']
+            params['servers'] = self.parameters['nameservers']
+            params['svm'] = {'name': self.parameters['vserver']}
+            message, error = self.restApi.post(api, params)
+            if error:
+                self.module.fail_json(msg=error)
+        else:
             dns = netapp_utils.zapi.NaElement('net-dns-create')
             nameservers = netapp_utils.zapi.NaElement('name-servers')
             domains = netapp_utils.zapi.NaElement('domains')
@@ -132,11 +156,19 @@ class NetAppOntapDns(object):
                                           (to_native(error)),
                                       exception=traceback.format_exc())
 
-    def destroy_dns(self):
+    def destroy_dns(self, dns_attrs):
         """
         Destroys an already created dns
         :return:
         """
+        if self.use_rest:
+            uuid = dns_attrs['records'][0]['svm']['uuid']
+            api = 'name-services/dns/' + uuid
+            data = None
+            message, error = self.restApi.delete(api, data)
+            if error:
+                self.module.fail_json(msg=error)
+        else:
             try:
                 self.server.invoke_successfully(netapp_utils.zapi.NaElement('net-dns-destroy'), True)
             except netapp_utils.zapi.NaApiError as error:
@@ -145,6 +177,22 @@ class NetAppOntapDns(object):
                                       exception=traceback.format_exc())
 
     def get_dns(self):
+        if self.use_rest:
+            api = "name-services/dns"
+            params = {'fields': 'domains,servers,svm',
+                      "svm.name": self.parameters['vserver']}
+            message, error = self.restApi.get(api, params)
+            if error:
+                self.module.fail_json(msg=error)
+            if len(message.keys()) == 0:
+                message = None
+            elif 'records' in message and len(message['records']) == 0:
+                message = None
+            elif 'records' not in message or len(message['records']) != 1:
+                error = "Unexpected response from %s: %s" % (api, repr(message))
+                self.module.fail_json(msg=error)
+            return message
+        else:
             dns_obj = netapp_utils.zapi.NaElement('net-dns-get')
             try:
                 result = self.server.invoke_successfully(dns_obj, True)
@@ -168,6 +216,23 @@ class NetAppOntapDns(object):
             return attrs
 
     def modify_dns(self, dns_attrs):
+        if self.use_rest:
+            changed = False
+            params = {}
+            if dns_attrs['records'][0]['servers'] != self.parameters['nameservers']:
+                changed = True
+                params['servers'] = self.parameters['nameservers']
+            if dns_attrs['records'][0]['domains'] != self.parameters['domains']:
+                changed = True
+                params['domains'] = self.parameters['domains']
+            if changed:
+                uuid = dns_attrs['records'][0]['svm']['uuid']
+                api = "name-services/dns/" + uuid
+                message, error = self.restApi.patch(api, params)
+                if error:
+                    self.module.fail_json(msg=error)
+
+        else:
             changed = False
             dns = netapp_utils.zapi.NaElement('net-dns-modify')
             if dns_attrs['nameservers'] != self.parameters['nameservers']:
@@ -200,8 +265,8 @@ class NetAppOntapDns(object):
 
     def apply(self):
         # asup logging
+        if not self.use_rest:
             netapp_utils.ems_log_event("na_ontap_dns", self.server)
-
         dns_attrs = self.get_dns()
         changed = False
         if self.parameters['state'] == 'present':
@@ -212,7 +277,7 @@ class NetAppOntapDns(object):
                 changed = True
         else:
             if dns_attrs is not None:
-                self.destroy_dns()
+                self.destroy_dns(dns_attrs)
                 changed = True
         self.module.exit_json(changed=changed)
 

lib/ansible/plugins/doc_fragments/netapp.py

@@ -196,3 +196,28 @@ notes:
   - Embedded Web Services is currently available on the E2800, E5700, EF570, and newer hardware models.
   - M(netapp_e_storage_system) may be utilized for configuring the systems managed by a WSP instance.
 '''
+
+    # Documentation fragment for AWSCVS
+    AWSCVS = """
+options:
+  api_key:
+    required: true
+    description:
+    - The access key to authenticate with the AWSCVS Web Services Proxy or Embedded Web Services API.
+  secret_key:
+    required: true
+    description:
+    - The secret_key to authenticate with the AWSCVS Web Services Proxy or Embedded Web Services API.
+  api_url:
+    required: true
+    description:
+    - The url to the AWSCVS Web Services Proxy or Embedded Web Services API.
+  validate_certs:
+    required: false
+    default: true
+    description:
+    - Should https certificates be validated?
+    type: bool
+notes:
+  - The modules prefixed with aws\\_cvs\\_netapp are built to Manage AWS Cloud Volume Service .
+"""

test/units/modules/storage/netapp/test_na_ontap_dns.py

@@ -0,0 +1,276 @@
+# (c) 2018, NetApp, Inc
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+''' unit tests for Ansible module: na_ontap_dns'''
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+import json
+import pytest
+
+from units.compat import unittest
+from units.compat.mock import patch, Mock
+from ansible.module_utils import basic
+from ansible.module_utils._text import to_bytes
+import ansible.module_utils.netapp as netapp_utils
+
+from ansible.modules.storage.netapp.na_ontap_dns \
+    import NetAppOntapDns as dns_module  # module under test
+
+if not netapp_utils.has_netapp_lib():
+    pytestmark = pytest.mark.skip('skipping as missing required netapp_lib')
+HAS_NETAPP_ZAPI_MSG = "pip install netapp_lib is required"
+
+
+# REST API canned responses when mocking send_request
+SRR = {
+    # common responses
+    'is_rest': (200, None),
+    'is_zapi': (400, "Unreachable"),
+    'empty_good': ({}, None),
+    'end_of_sequence': (None, "Unexpected call to send_request"),
+    'generic_error': (None, "Expected error"),
+    'dns_record': ({"records": [{"domains": ['0.0.0.0'],
+                                 "servers": ['0.0.0.0'],
+                                 "svm": {"name": "svm1", "uuid": "02c9e252-41be-11e9-81d5-00a0986138f7"}}]}, None)}
+
+
+def set_module_args(args):
+    """prepare arguments so that they will be picked up during module creation"""
+    args = json.dumps({'ANSIBLE_MODULE_ARGS': args})
+    basic._ANSIBLE_ARGS = to_bytes(args)  # pylint: disable=protected-access
+
+
+class AnsibleExitJson(Exception):
+    """Exception class to be raised by module.exit_json and caught by the test case"""
+    pass
+
+
+class AnsibleFailJson(Exception):
+    """Exception class to be raised by module.fail_json and caught by the test case"""
+    pass
+
+
+def exit_json(*args, **kwargs):  # pylint: disable=unused-argument
+    """function to patch over exit_json; package return data into an exception"""
+    if 'changed' not in kwargs:
+        kwargs['changed'] = False
+    raise AnsibleExitJson(kwargs)
+
+
+def fail_json(*args, **kwargs):  # pylint: disable=unused-argument
+    """function to patch over fail_json; package return data into an exception"""
+    kwargs['failed'] = True
+    raise AnsibleFailJson(kwargs)
+
+
+class MockONTAPConnection(object):
+    ''' mock server connection to ONTAP host '''
+
+    def __init__(self, kind=None, data=None):
+        ''' save arguments '''
+        self.kind = kind
+        self.params = data
+        self.xml_in = None
+        self.xml_out = None
+
+    def invoke_successfully(self, xml, enable_tunneling):  # pylint: disable=unused-argument
+        ''' mock invoke_successfully returning xml data '''
+        self.xml_in = xml
+        request = xml.to_string().decode('utf-8')
+        if request.startswith("<ems-autosupport-log>"):
+            xml = None  # or something that may the logger happy, and you don't need @patch anymore
+            # or
+            # xml = build_ems_log_response()
+        elif request == "<net-dns-get/>":
+            if self.kind == 'create':
+                raise netapp_utils.zapi.NaApiError(code="15661")
+            else:
+                xml = self.build_dns_status_info()
+        elif request.startswith("<net-dns-create>"):
+            xml = self.build_dns_status_info()
+        if self.kind == 'enable':
+            xml = self.build_dns_status_info()
+        self.xml_out = xml
+        return xml
+
+    @staticmethod
+    def build_dns_status_info():
+        xml = netapp_utils.zapi.NaElement('xml')
+        nameservers = [{'ip-address': '0.0.0.0'}]
+        domains = [{'string': '0.0.0.0'}]
+        attributes = {'num-records': 1,
+                      'attributes': {'net-dns-info': {'name-servers': nameservers,
+                                                      'domains': domains,
+                                                      'skip-config-validation': 'false'}}}
+        xml.translate_struct(attributes)
+        return xml
+
+
+class TestMyModule(unittest.TestCase):
+    ''' Unit tests for na_ontap_job_schedule '''
+
+    def setUp(self):
+        self.mock_module_helper = patch.multiple(basic.AnsibleModule,
+                                                 exit_json=exit_json,
+                                                 fail_json=fail_json)
+        self.mock_module_helper.start()
+        self.addCleanup(self.mock_module_helper.stop)
+
+    def mock_args(self):
+        return {
+            'state': 'present',
+            'vserver': 'vserver',
+            'nameservers': ['0.0.0.0'],
+            'domains': ['0.0.0.0'],
+            'hostname': 'test',
+            'username': 'test_user',
+            'password': 'test_pass!'
+        }
+
+    def get_dns_mock_object(self, type='zapi', kind=None, status=None):
+        dns_obj = dns_module()
+        if type == 'zapi':
+            if kind is None:
+                dns_obj.server = MockONTAPConnection()
+            else:
+                dns_obj.server = MockONTAPConnection(kind=kind, data=status)
+        return dns_obj
+
+    def test_module_fail_when_required_args_missing(self):
+        ''' required arguments are reported as errors '''
+        with pytest.raises(AnsibleFailJson) as exc:
+            set_module_args({})
+            dns_module()
+        print('Info: %s' % exc.value.args[0]['msg'])
+
+    def test_idempotent_modify_dns(self):
+        data = self.mock_args()
+        set_module_args(data)
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object('zapi', 'enable', 'false').apply()
+        assert not exc.value.args[0]['changed']
+
+    def test_successfully_modify_dns(self):
+        data = self.mock_args()
+        data['domains'] = ['1.1.1.1']
+        set_module_args(data)
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object('zapi', 'enable', 'false').apply()
+        assert exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.ems_log_event')
+    def test_idempotent_create_dns(self, mock_ems_log_event):
+        data = self.mock_args()
+        set_module_args(data)
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object('zapi').apply()
+        assert not exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.ems_log_event')
+    def test_successfully_create_dns(self, mock_ems_log_event):
+        data = self.mock_args()
+        print("create dns")
+        data['domains'] = ['1.1.1.1']
+        set_module_args(data)
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object('zapi', 'create').apply()
+        assert exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_error(self, mock_request):
+        data = self.mock_args()
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['generic_error'],
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleFailJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert exc.value.args[0]['msg'] == SRR['generic_error'][1]
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_successfully_create(self, mock_request):
+        data = self.mock_args()
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['empty_good'],  # get
+            SRR['empty_good'],  # post
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_idempotent_create_dns(self, mock_request):
+        data = self.mock_args()
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['dns_record'],  # get
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert not exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_successfully_destroy(self, mock_request):
+        data = self.mock_args()
+        data['state'] = 'absent'
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['dns_record'],  # get
+            SRR['empty_good'],  # delete
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_idempotently_destroy(self, mock_request):
+        data = self.mock_args()
+        data['state'] = 'absent'
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['empty_good'],  # get
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert not exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_successfully_modify(self, mock_request):
+        data = self.mock_args()
+        data['state'] = 'present'
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['empty_good'],  # get
+            SRR['empty_good'],  # patch
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert exc.value.args[0]['changed']
+
+    @patch('ansible.module_utils.netapp.OntapRestAPI.send_request')
+    def test_rest_idempotently_modify(self, mock_request):
+        data = self.mock_args()
+        data['state'] = 'present'
+        set_module_args(data)
+        mock_request.side_effect = [
+            SRR['is_rest'],
+            SRR['dns_record'],  # get
+            SRR['end_of_sequence']
+        ]
+        with pytest.raises(AnsibleExitJson) as exc:
+            self.get_dns_mock_object(type='rest').apply()
+        assert not exc.value.args[0]['changed']