From ec1c5585af424f4c43fa7a16cbf82c2571045415 Mon Sep 17 00:00:00 2001
From: Chris Van Heuveln <cvanheuv@cisco.com>
Date: Thu, 31 Oct 2019 08:47:39 -0400
Subject: [PATCH] nxos_vrf: N6K regression test failures (#63277)

* nxos_vrf: N6K regression test needs 'nv overlay evpn'

* Add delay for vrf creation

* Revert "Add delay for vrf creation"

This reverts commit 8afa5281ff4b3c5a1775fbd40c5acb7ab5ecdf68.

* Add vrf_error_check w/delay to module

* whitespace

* httpapi fixes

* /input/input_data/ - 'input' is a built-in

* shippable retry
---
 lib/ansible/module_utils/network/nxos/nxos.py    |  2 ++
 lib/ansible/modules/network/nxos/nxos_vrf.py     | 16 +++++++++++++++-
 lib/ansible/plugins/httpapi/nxos.py              |  6 +++++-
 .../targets/nxos_vrf/tests/common/sanity.yaml    | 16 ++++++++++++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/lib/ansible/module_utils/network/nxos/nxos.py b/lib/ansible/module_utils/network/nxos/nxos.py
index 659ae54934b..0b9c7eea28c 100644
--- a/lib/ansible/module_utils/network/nxos/nxos.py
+++ b/lib/ansible/module_utils/network/nxos/nxos.py
@@ -603,6 +603,8 @@ class HttpApi:
             if opts.get('ignore_timeout') and code:
                 responses.append(code)
                 return responses
+            elif opts.get('catch_clierror') and '400' in code:
+                return [code, err]
             elif code and 'no graceful-restart' in err:
                 if 'ISSU/HA will be affected if Graceful Restart is disabled' in err:
                     msg = ['']
diff --git a/lib/ansible/modules/network/nxos/nxos_vrf.py b/lib/ansible/modules/network/nxos/nxos_vrf.py
index 5e95d18a35c..5252c7e437f 100644
--- a/lib/ansible/modules/network/nxos/nxos_vrf.py
+++ b/lib/ansible/modules/network/nxos/nxos_vrf.py
@@ -457,6 +457,19 @@ def check_declarative_intent_params(want, module, element_spec, result):
                     module.fail_json(msg="Interface %s not configured on vrf %s" % (i, w['name']))
 
 
+def vrf_error_check(module, commands, responses):
+    """Checks for VRF config errors and executes a retry in some circumstances.
+    """
+    pattern = 'ERROR: Deletion of VRF .* in progress'
+    if re.search(pattern, str(responses)):
+        # Allow delay/retry for VRF changes
+        time.sleep(15)
+        responses = load_config(module, commands, opts={'catch_clierror': True})
+        if re.search(pattern, str(responses)):
+            module.fail_json(msg='VRF config (and retry) failure: %s ' % responses)
+        module.warn('VRF config delayed by VRF deletion - passed on retry')
+
+
 def main():
     """ main entry point for module execution
     """
@@ -504,7 +517,8 @@ def main():
     result['commands'] = commands
 
     if commands and not module.check_mode:
-        load_config(module, commands)
+        responses = load_config(module, commands, opts={'catch_clierror': True})
+        vrf_error_check(module, commands, responses)
         result['changed'] = True
 
     check_declarative_intent_params(want, module, element_spec, result)
diff --git a/lib/ansible/plugins/httpapi/nxos.py b/lib/ansible/plugins/httpapi/nxos.py
index 013cc4a67ea..949c34eec89 100644
--- a/lib/ansible/plugins/httpapi/nxos.py
+++ b/lib/ansible/plugins/httpapi/nxos.py
@@ -178,7 +178,11 @@ def handle_response(response):
     if response['ins_api'].get('outputs'):
         for output in to_list(response['ins_api']['outputs']['output']):
             if output['code'] != '200':
-                raise ConnectionError('%s: %s' % (output['input'], output['msg']), code=output['code'])
+                # Best effort messages: some API output keys may not exist on some platforms
+                input_data = output.get('input', '')
+                msg = output.get('msg', '')
+                clierror = output.get('clierror', '')
+                raise ConnectionError('%s: %s: %s' % (input_data, msg, clierror), code=output['code'])
             elif 'body' in output:
                 result = output['body']
                 if isinstance(result, dict):
diff --git a/test/integration/targets/nxos_vrf/tests/common/sanity.yaml b/test/integration/targets/nxos_vrf/tests/common/sanity.yaml
index ebf6ea3ec92..a39883cab8f 100644
--- a/test/integration/targets/nxos_vrf/tests/common/sanity.yaml
+++ b/test/integration/targets/nxos_vrf/tests/common/sanity.yaml
@@ -25,6 +25,14 @@
     state: enabled
   ignore_errors: yes
 
+- name: "Setup: Enable nv overlay evpn"
+  nxos_config:
+    commands:
+      - nv overlay evpn
+    provider: "{{ connection }}"
+  ignore_errors: yes
+  when: platform is match("N5K|N6K")
+
 - block:
   - name: Ensure ntc VRF exists on switch
     nxos_vrf: &configure
@@ -91,6 +99,14 @@
   - assert: *false
 
   always:
+  - name: "Setup: Disable nv overlay evpn"
+    nxos_config:
+      commands:
+        - no nv overlay evpn
+      provider: "{{ connection }}"
+    ignore_errors: yes
+    when: platform is match("N5K|N6K")
+
   - name: "Disable feature BGP"
     nxos_feature:
       feature: bgp