cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[cloud] Retry WAF actions on WAFStaleDataException (#36405)

Browse source 
Add a util to run functions with AWSRetry to retry on WAFStaleDataExceptions and update ChangeToken for each attempt
This commit is contained in:
Sloane Hertel 2018-02-21 08:14:17 -05:00 committed by Ryan Brown
parent 9598978e12
commit f7d79d4789
5 changed files with 65 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/ansible/module_utils/aws/waf.py
View file

@ -180,3 +180,9 @@ def get_change_token(client, module):
return token['ChangeToken']
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg="Couldn't obtain change token")
@AWSRetry.backoff(tries=10, delay=2, backoff=2.0, catch_extra_error_codes=['WAFStaleDataException'])
def run_func_with_change_token_backoff(client, module, params, func):
params['ChangeToken'] = get_change_token(client, module)
return func(**params)

34
lib/ansible/modules/cloud/amazon/aws_waf_condition.py
View file

@ -331,7 +331,7 @@ except ImportError:
from ansible.module_utils.aws.core import AnsibleAWSModule
from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec
from ansible.module_utils.ec2 import camel_dict_to_snake_dict, AWSRetry, compare_policies
from ansible.module_utils.aws.waf import get_change_token, MATCH_LOOKUP
from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, MATCH_LOOKUP
from ansible.module_utils.aws.waf import get_rule_with_backoff, list_rules_with_backoff
@ -397,12 +397,10 @@ class Condition(object):
kwargs['Updates'].append({'Action': 'INSERT', self.conditiontuple: condition_insert})
kwargs[self.conditionsetid] = condition_set_id
kwargs['ChangeToken'] = get_change_token(self.client, self.module)
return kwargs
def format_for_deletion(self, condition):
return {'ChangeToken': get_change_token(self.client, self.module),
'Updates': [{'Action': 'DELETE', self.conditiontuple: current_condition_tuple}
return {'Updates': [{'Action': 'DELETE', self.conditiontuple: current_condition_tuple}
for current_condition_tuple in condition[self.conditiontuples]],
self.conditionsetid: condition[self.conditionsetid]}
@ -443,15 +441,17 @@ class Condition(object):
pattern_set = self.get_regex_pattern_by_name(name)
if not pattern_set:
pattern_set = self.client.create_regex_pattern_set(Name=name, ChangeToken=get_change_token(self.client, self.module))['RegexPatternSet']
pattern_set = run_func_with_change_token_backoff(self.client, self.module, {'Name': name},
self.client.create_regex_pattern_set)['RegexPatternSet']
missing = set(regex_pattern['regex_strings']) - set(pattern_set['RegexPatternStrings'])
extra = set(pattern_set['RegexPatternStrings']) - set(regex_pattern['regex_strings'])
if not missing and not extra:
return pattern_set
updates = [{'Action': 'INSERT', 'RegexPatternString': pattern} for pattern in missing]
updates.extend([{'Action': 'DELETE', 'RegexPatternString': pattern} for pattern in extra])
self.client.update_regex_pattern_set(RegexPatternSetId=pattern_set['RegexPatternSetId'],
Updates=updates, ChangeToken=get_change_token(self.client, self.module))
run_func_with_change_token_backoff(self.client, self.module,
{'RegexPatternSetId': pattern_set['RegexPatternSetId'], 'Updates': updates},
self.client.update_regex_pattern_set)
return self.get_regex_pattern_set_with_backoff(pattern_set['RegexPatternSetId'])['RegexPatternSet']
def delete_unused_regex_pattern(self, regex_pattern_set_id):
@ -460,11 +460,13 @@ class Condition(object):
updates = list()
for regex_pattern_string in regex_pattern_set['RegexPatternStrings']:
updates.append({'Action': 'DELETE', 'RegexPatternString': regex_pattern_string})
self.client.update_regex_pattern_set(RegexPatternSetId=regex_pattern_set_id, Updates=updates,
ChangeToken=get_change_token(self.client, self.module))
run_func_with_change_token_backoff(self.client, self.module,
{'RegexPatternSetId': regex_pattern_set_id, 'Updates': updates},
self.client.update_regex_pattern_set)
self.client.delete_regex_pattern_set(RegexPatternSetId=regex_pattern_set_id,
ChangeToken=get_change_token(self.client, self.module))
run_func_with_change_token_backoff(self.client, self.module,
{'RegexPatternSetId': regex_pattern_set_id},
self.client.delete_regex_pattern_set)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg='Could not delete regex pattern')
@ -535,15 +537,14 @@ class Condition(object):
func = getattr(self.client, 'update_' + self.method_suffix)
params = self.format_for_deletion(current_condition)
try:
func(**params)
run_func_with_change_token_backoff(self.client, self.module, params, func)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg='Could not delete filters from condition')
func = getattr(self.client, 'delete_' + self.method_suffix)
params = dict()
params[self.conditionsetid] = condition_set_id
params['ChangeToken'] = get_change_token(self.client, self.module)
try:
func(**params)
run_func_with_change_token_backoff(self.client, self.module, params, func)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg='Could not delete condition')
# tidy up regex patterns
@ -579,7 +580,7 @@ class Condition(object):
update['Updates'] = missing + extra
func = getattr(self.client, 'update_' + self.method_suffix)
try:
func(**update)
run_func_with_change_token_backoff(self.client, self.module, update, func)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg='Could not update condition')
return changed, self.get_condition_by_id(condition_set_id)
@ -592,10 +593,9 @@ class Condition(object):
else:
params = dict()
params['Name'] = name
params['ChangeToken'] = get_change_token(self.client, self.module)
func = getattr(self.client, 'create_' + self.method_suffix)
try:
condition = func(**params)
condition = run_func_with_change_token_backoff(self.client, self.module, params, func)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg='Could not create condition')
return self.find_and_update_condition(condition[self.conditionset][self.conditionsetid])

17
lib/ansible/modules/cloud/amazon/aws_waf_rule.py
View file

@ -126,7 +126,7 @@ except ImportError:
from ansible.module_utils.aws.core import AnsibleAWSModule
from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec
from ansible.module_utils.ec2 import camel_dict_to_snake_dict
from ansible.module_utils.aws.waf import get_change_token, list_rules_with_backoff, MATCH_LOOKUP
from ansible.module_utils.aws.waf import run_func_with_change_token_backoff, list_rules_with_backoff, MATCH_LOOKUP
from ansible.module_utils.aws.waf import get_web_acl_with_backoff, list_web_acls_with_backoff
@ -201,10 +201,13 @@ def find_and_update_rule(client, module, rule_id):
if not all_conditions[condition_type][condition['data_id']]['name'] in desired_conditions[condition_type]])
changed = bool(insertions or deletions)
update = {
'RuleId': rule_id,
'Updates': insertions + deletions
}
if changed:
try:
client.update_rule(RuleId=rule_id, ChangeToken=get_change_token(client, module),
Updates=insertions + deletions)
run_func_with_change_token_backoff(client, module, update, client.update_rule)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not update rule conditions')
@ -229,8 +232,7 @@ def remove_rule_conditions(client, module, rule_id):
conditions = get_rule(client, module, rule_id)['Predicates']
updates = [format_for_deletion(camel_dict_to_snake_dict(condition)) for condition in conditions]
try:
client.update_rule(RuleId=rule_id,
ChangeToken=get_change_token(client, module), Updates=updates)
run_func_with_change_token_backoff(client, module, {'RuleId': rule_id, 'Updates': updates}, client.update_rule)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not remove rule conditions')
@ -247,9 +249,8 @@ def ensure_rule_present(client, module):
if not metric_name:
metric_name = re.sub(r'[^a-zA-Z0-9]', '', module.params['name'])
params['MetricName'] = metric_name
params['ChangeToken'] = get_change_token(client, module)
try:
new_rule = client.create_rule(**params)['Rule']
new_rule = run_func_with_change_token_backoff(client, module, params, client.create_rule)['Rule']
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not create rule')
return find_and_update_rule(client, module, new_rule['RuleId'])
@ -281,7 +282,7 @@ def ensure_rule_absent(client, module):
if rule_id:
remove_rule_conditions(client, module, rule_id)
try:
return True, client.delete_rule(RuleId=rule_id, ChangeToken=get_change_token(client, module))
return True, run_func_with_change_token_backoff(client, module, {'RuleId': rule_id}, client.delete_rule)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not delete rule')
return False, {}

37
lib/ansible/modules/cloud/amazon/aws_waf_web_acl.py
View file

@ -136,7 +136,7 @@ import re
from ansible.module_utils.aws.core import AnsibleAWSModule
from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, ec2_argument_spec, camel_dict_to_snake_dict
from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, get_change_token
from ansible.module_utils.aws.waf import list_rules_with_backoff, list_web_acls_with_backoff, run_func_with_change_token_backoff
def get_web_acl_by_name(client, module, name):
@ -186,16 +186,26 @@ def find_and_update_web_acl(client, module, web_acl_id):
insertions = [format_for_update(rule, 'INSERT') for rule in missing]
deletions = [format_for_update(rule, 'DELETE') for rule in extras]
changed = bool(insertions + deletions)
if changed:
# Purge rules before adding new ones in case a deletion shares the same
# priority as an insertion.
params = {
'WebACLId': acl['WebACLId'],
'DefaultAction': acl['DefaultAction']
}
if deletions:
try:
client.update_web_acl(
WebACLId=acl['WebACLId'],
ChangeToken=get_change_token(client, module),
Updates=insertions + deletions,
DefaultAction=acl['DefaultAction']
)
params['Updates'] = deletions
run_func_with_change_token_backoff(client, module, params, client.update_web_acl)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not update Web ACL')
if insertions:
try:
params['Updates'] = insertions
run_func_with_change_token_backoff(client, module, params, client.update_web_acl)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not update Web ACL')
if changed:
acl = get_web_acl(client, module, web_acl_id)
return changed, acl
@ -217,8 +227,8 @@ def remove_rules_from_web_acl(client, module, web_acl_id):
acl = get_web_acl(client, module, web_acl_id)
deletions = [format_for_update(rule, 'DELETE') for rule in acl['Rules']]
try:
client.update_web_acl(WebACLId=acl['WebACLId'], ChangeToken=get_change_token(client, module),
Updates=deletions, DefaultAction=acl['DefaultAction'])
params = {'WebACLId': acl['WebACLId'], 'DefaultAction': acl['DefaultAction'], 'Updates': deletions}
run_func_with_change_token_backoff(client, module, params, client.update_web_acl)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not remove rule')
@ -236,9 +246,8 @@ def ensure_web_acl_present(client, module):
metric_name = re.sub(r'[^A-Za-z0-9]', '', module.params['name'])
default_action = module.params['default_action'].upper()
try:
new_web_acl = client.create_web_acl(Name=name, MetricName=metric_name,
DefaultAction={'Type': default_action},
ChangeToken=get_change_token(client, module))
params = {'Name': name, 'MetricName': metric_name, 'DefaultAction': {'Type': default_action}}
new_web_acl = run_func_with_change_token_backoff(client, module, params, client.create_web_acl)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not create Web ACL')
(changed, result) = find_and_update_web_acl(client, module, new_web_acl['WebACL']['WebACLId'])
@ -252,7 +261,7 @@ def ensure_web_acl_absent(client, module):
if web_acl['Rules']:
remove_rules_from_web_acl(client, module, web_acl_id)
try:
client.delete_web_acl(WebACLId=web_acl_id, ChangeToken=get_change_token(client, module))
run_func_with_change_token_backoff(client, module, {'WebACLId': web_acl_id}, client.delete_web_acl)
return True, {}
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
module.fail_json_aws(e, msg='Could not delete Web ACL')

10
test/integration/targets/aws_waf_web_acl/tasks/main.yml
View file

@ -481,10 +481,19 @@
- debug:
msg: "****** TEARDOWN STARTS HERE ******"
- name: delete the web acl
aws_waf_web_acl:
name: "{{ resource_prefix }}_web_acl"
state: absent
purge_rules: yes
<<: *aws_connection_info
ignore_errors: yes
- name: remove second WAF rule
aws_waf_rule:
name: "{{ resource_prefix }}_rule_2"
state: absent
purge_conditions: yes
<<: *aws_connection_info
ignore_errors: yes
@ -492,6 +501,7 @@
aws_waf_rule:
name: "{{ resource_prefix }}_rule"
state: absent
purge_conditions: yes
<<: *aws_connection_info
ignore_errors: yes