ansible/test/integration/targets/aws_ses_rule_set/tasks/obtain-lock.yaml
Ed Costello b70d5d9aee [AWS] ses rule set module for inbound email processing (#42781)
* Add module ses_rule_set for Amazon SES

* Update behaviours and naming to be consistent with other aws_ses_ modules.

* Add global lock around tests using active rule sets to prevent intermittent test failures.

* Fix deletion of rule sets so that we don't inactivate the active rule set
when force deleting an inactive rule set.
2018-11-14 12:15:24 -05:00

126 lines
5 KiB
YAML

# ============================================================
# This file attempts to obtain a global lock (for a given
# region / account combination.
#
# This makes one attempt to get the lock and will set the
# won_lock variable to True or False to indicate whether
# or not we got the lock.
#
# It's expected that this will be executed in a retry loop
# so that if we don't get the lock we delay then try again.
#
# This should only be used in a block with cleanup-lock.yaml
# included in the always clause to ensure the lock is released.
#
# There are several variables that control the locking behaviour:
# * lock_timeout_seconds
# How old a lock must be before it's assumed to be an expired
# lock that was not cleaned up by the owner. Any locks older
# than this will not prevent a lock being obtained and will
# be deleted when a new process obtains the lock.
# * lock_log_group_prefix
# The log_group prefix that represents the lock being obtained.
# This must be the same across all processes trying to obtain
# the lock.
# * lock_process_id
# A unique identifier of this process. Each process that might
# attempt to lock the process must have a different identifier.
# This defaults to the resource_prefix which is generally
# appropriate.
# * max_obtain_lock_attempts
# How many attempts to make to get the lock before giving up
# NB: This is actually done in main.yaml
# * obtain_lock_delay_seconds:
# How long to delay after failing to get the lock before
# trying again.
# NB: This is actually done in obtain-lock-wrapper.yaml
#
# The locking here is based around creating cloudwatch log groups.
# This resource was chosen because:
# A) it's free
# B) we have a built in grouping concept because of the hierarchy
# that allows us to easily group attempts for the same lock
# C) the creation time is tracked and returned which gives us
# a mechanism for deterministically picking a winner
#
# Each lock is represented by a log group prefix. Each attempt
# to obtain the lock is a log group of the lock_process_id below
# that prefix.
#
# The winning lock is the one with the earliest creation time.
#
# To prevent a hanging lock from permanently hanging the build
# lock attempts older than the lock timeout are ignored and
# cleaned up by the next process to win the lock.
# ============================================================
- name: set up aws connection info
set_fact:
aws_connection_info: &aws_connection_info
aws_access_key: "{{ aws_access_key }}"
aws_secret_key: "{{ aws_secret_key }}"
security_token: "{{ security_token }}"
region: "{{ aws_region }}"
no_log: yes
- name: Set lock_attempt_log_group_name
set_fact:
lock_attempt_log_group_name: "{{ lock_log_group_prefix }}/{{ lock_process_id|default(resource_prefix) }}"
# Note the overwrite below to ensure that the creation time
# is upated. This is important as we calculate expiry relative
# the attempt creation.
#
# Because of this it's imporatnt that we delete the attempt
# if we don't get the lock. Otherwise we can get a deadlock
# where the stale atttempt from one process wins, but then
# because that process updates the creation date it doesn't
# consider its self to havewone.
- name: Create Lock Attempt Log Group
cloudwatchlogs_log_group:
log_group_name: "{{ lock_attempt_log_group_name }}"
state: present
overwrite: True
<<: *aws_connection_info
register: lock_attempt_log_group_result
- name: Get Lock Attempt Lock Groups
cloudwatchlogs_log_group_facts:
log_group_name: "{{ lock_log_group_prefix }}/"
<<: *aws_connection_info
register: lock_attempt_log_groups
- name: Calculate Expired Lock Attempt Timestamp
set_fact:
expired_lock_timestamp: "{{ lock_attempt_log_group_result.creation_time - (lock_timeout_seconds * 1000) }}"
- name: Get Expired and Active Lock Attempts
set_fact:
expired_lock_attempts: "{{ lock_attempt_log_groups.log_groups|selectattr('creation_time', 'lt', expired_lock_timestamp|int)|list }}"
active_lock_attempts: "{{ lock_attempt_log_groups.log_groups|selectattr('creation_time', 'ge', expired_lock_timestamp|int)|list }}"
- name: Pick Winning Lock Attempt
set_fact:
winning_lock_attempt: "{{ active_lock_attempts|sort(attribute='creation_time')|first }}"
- name: Determine if Won Lock
set_fact:
won_lock: "{{ winning_lock_attempt.log_group_name == lock_attempt_log_group_name }}"
# Remove the lock attempt if we didn't get the lock. This prevents
# our stale lock attempt blocking another process from getting the lock.
# See more detailed comment above Create Lock Attempt Log Group
- name: Remove Failed Lock Attempt Log Group
cloudwatchlogs_log_group:
log_group_name: "{{ lock_attempt_log_group_name }}"
state: absent
<<: *aws_connection_info
when: "not won_lock|bool"
- name: Delete Expired Lock Attempts
cloudwatchlogs_log_group:
log_group_name: "{{ item.log_group_name }}"
state: absent
<<: *aws_connection_info
when: "won_lock|bool"
loop: "{{ expired_lock_attempts }}"