efs_facts: improve performance by reducing the number of api calls (#36520)
* efs_facts: improve performance by reducing the number of api calls * Remove efs_facts tests from running in CI
This commit is contained in:
parent
0e6628395a
commit
0f612d1b76
4 changed files with 311 additions and 16 deletions
36
hacking/aws_config/testing_policies/efs-policy.json
Normal file
36
hacking/aws_config/testing_policies/efs-policy.json
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Sid": "ManageNetwork",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"ec2:CreateNetworkInterface",
|
||||||
|
"ec2:CreateSubnet",
|
||||||
|
"ec2:CreateTags",
|
||||||
|
"ec2:CreateVpc",
|
||||||
|
"ec2:DeleteNetworkInterface",
|
||||||
|
"ec2:DeleteSubnet",
|
||||||
|
"ec2:DeleteVpc",
|
||||||
|
"ec2:DescribeNetworkInterfaceAttribute",
|
||||||
|
"ec2:DescribeNetworkInterfaces",
|
||||||
|
"ec2:DescribeSecurityGroups",
|
||||||
|
"ec2:DescribeSubnets",
|
||||||
|
"ec2:DescribeTags",
|
||||||
|
"ec2:DescribeVpcAttribute",
|
||||||
|
"ec2:DescribeVpcClassicLink",
|
||||||
|
"ec2:DescribeVpcs",
|
||||||
|
"ec2:ModifyVpcAttribute"
|
||||||
|
],
|
||||||
|
"Resource": "*"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Sid": "ManageEFS",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"elasticfilesystem:*"
|
||||||
|
],
|
||||||
|
"Resource": "*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
|
@ -209,6 +209,33 @@ class EFSConnection(object):
|
||||||
"""
|
"""
|
||||||
return self.connection.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups']
|
return self.connection.describe_mount_target_security_groups(MountTargetId=mount_target_id)['SecurityGroups']
|
||||||
|
|
||||||
|
def get_mount_targets_data(self, file_systems):
|
||||||
|
for item in file_systems:
|
||||||
|
if item['life_cycle_state'] == self.STATE_AVAILABLE:
|
||||||
|
try:
|
||||||
|
mount_targets = self.get_mount_targets(item['file_system_id'])
|
||||||
|
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
|
||||||
|
self.module.fail_json_aws(e, msg="Couldn't get EFS targets")
|
||||||
|
for mt in mount_targets:
|
||||||
|
item['mount_targets'].append(camel_dict_to_snake_dict(mt))
|
||||||
|
return file_systems
|
||||||
|
|
||||||
|
def get_security_groups_data(self, file_systems):
|
||||||
|
for item in file_systems:
|
||||||
|
if item['life_cycle_state'] == self.STATE_AVAILABLE:
|
||||||
|
for target in item['mount_targets']:
|
||||||
|
if target['life_cycle_state'] == self.STATE_AVAILABLE:
|
||||||
|
try:
|
||||||
|
target['security_groups'] = self.get_security_groups(target['mount_target_id'])
|
||||||
|
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
|
||||||
|
self.module.fail_json_aws(e, msg="Couldn't get EFS security groups")
|
||||||
|
else:
|
||||||
|
target['security_groups'] = []
|
||||||
|
else:
|
||||||
|
item['tags'] = {}
|
||||||
|
item['mount_targets'] = []
|
||||||
|
return file_systems
|
||||||
|
|
||||||
def get_file_systems(self, file_system_id=None, creation_token=None):
|
def get_file_systems(self, file_system_id=None, creation_token=None):
|
||||||
kwargs = dict()
|
kwargs = dict()
|
||||||
if file_system_id:
|
if file_system_id:
|
||||||
|
@ -230,23 +257,9 @@ class EFSConnection(object):
|
||||||
item['MountPoint'] = '.%s.efs.%s.amazonaws.com:/' % (item['FileSystemId'], self.region)
|
item['MountPoint'] = '.%s.efs.%s.amazonaws.com:/' % (item['FileSystemId'], self.region)
|
||||||
if 'Timestamp' in item['SizeInBytes']:
|
if 'Timestamp' in item['SizeInBytes']:
|
||||||
item['SizeInBytes']['Timestamp'] = str(item['SizeInBytes']['Timestamp'])
|
item['SizeInBytes']['Timestamp'] = str(item['SizeInBytes']['Timestamp'])
|
||||||
if item['LifeCycleState'] == self.STATE_AVAILABLE:
|
|
||||||
try:
|
|
||||||
item['MountTargets'] = self.get_mount_targets(item['FileSystemId'])
|
|
||||||
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
|
|
||||||
self.module.fail_json_aws(e, msg="Couldn't get EFS targets")
|
|
||||||
for target in item['MountTargets']:
|
|
||||||
if target['LifeCycleState'] == self.STATE_AVAILABLE:
|
|
||||||
try:
|
|
||||||
target['SecurityGroups'] = self.get_security_groups(target['MountTargetId'])
|
|
||||||
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
|
|
||||||
self.module.fail_json_aws(e, msg="Couldn't get EFS security groups")
|
|
||||||
else:
|
|
||||||
target['SecurityGroups'] = []
|
|
||||||
else:
|
|
||||||
item['tags'] = {}
|
|
||||||
item['mount_targets'] = []
|
|
||||||
result = camel_dict_to_snake_dict(item)
|
result = camel_dict_to_snake_dict(item)
|
||||||
|
result['tags'] = {}
|
||||||
|
result['mount_targets'] = []
|
||||||
# Set tags *after* doing camel to snake
|
# Set tags *after* doing camel to snake
|
||||||
if result['life_cycle_state'] == self.STATE_AVAILABLE:
|
if result['life_cycle_state'] == self.STATE_AVAILABLE:
|
||||||
try:
|
try:
|
||||||
|
@ -340,6 +353,9 @@ def main():
|
||||||
if tags:
|
if tags:
|
||||||
file_systems_info = [item for item in file_systems_info if has_tags(item['tags'], tags)]
|
file_systems_info = [item for item in file_systems_info if has_tags(item['tags'], tags)]
|
||||||
|
|
||||||
|
file_systems_info = connection.get_mount_targets_data(file_systems_info)
|
||||||
|
file_systems_info = connection.get_security_groups_data(file_systems_info)
|
||||||
|
|
||||||
if targets:
|
if targets:
|
||||||
targets = [(item, prefix_to_attr(item)) for item in targets]
|
targets = [(item, prefix_to_attr(item)) for item in targets]
|
||||||
file_systems_info = [item for item in file_systems_info if has_targets(item['mount_targets'], targets)]
|
file_systems_info = [item for item in file_systems_info if has_targets(item['mount_targets'], targets)]
|
||||||
|
|
2
test/integration/targets/efs_facts/aliases
Normal file
2
test/integration/targets/efs_facts/aliases
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
cloud/aws
|
||||||
|
unsupported
|
241
test/integration/targets/efs_facts/tasks/main.yml
Normal file
241
test/integration/targets/efs_facts/tasks/main.yml
Normal file
|
@ -0,0 +1,241 @@
|
||||||
|
---
|
||||||
|
- block:
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: set connection information for all tasks
|
||||||
|
set_fact:
|
||||||
|
aws_connection_info: &aws_connection_info
|
||||||
|
aws_access_key: "{{ aws_access_key }}"
|
||||||
|
aws_secret_key: "{{ aws_secret_key }}"
|
||||||
|
security_token: "{{ security_token }}"
|
||||||
|
region: "{{ aws_region }}"
|
||||||
|
no_log: true
|
||||||
|
|
||||||
|
- name: Create VPC for testing
|
||||||
|
ec2_vpc_net:
|
||||||
|
name: "{{ resource_prefix }}-vpc"
|
||||||
|
cidr_block: 10.22.32.0/23
|
||||||
|
tags:
|
||||||
|
Name: Ansible ec2_instance Testing VPC
|
||||||
|
tenancy: default
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: testing_vpc
|
||||||
|
|
||||||
|
- name: Create subnet in zone A for testing
|
||||||
|
ec2_vpc_subnet:
|
||||||
|
state: present
|
||||||
|
vpc_id: "{{ testing_vpc.vpc.id }}"
|
||||||
|
cidr: 10.22.32.0/24
|
||||||
|
az: "{{ aws_region }}a"
|
||||||
|
resource_tags:
|
||||||
|
Name: "{{ resource_prefix }}-subnet-a"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: testing_subnet_a
|
||||||
|
|
||||||
|
- name: Create subnet in zone B for testing
|
||||||
|
ec2_vpc_subnet:
|
||||||
|
state: present
|
||||||
|
vpc_id: "{{ testing_vpc.vpc.id }}"
|
||||||
|
cidr: 10.22.33.0/24
|
||||||
|
az: "{{ aws_region }}b"
|
||||||
|
resource_tags:
|
||||||
|
Name: "{{ resource_prefix }}-subnet-b"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: testing_subnet_b
|
||||||
|
|
||||||
|
- name: Get default security group id for vpc
|
||||||
|
ec2_group_facts:
|
||||||
|
<<: *aws_connection_info
|
||||||
|
filters:
|
||||||
|
vpc-id: "{{ testing_vpc.vpc.id }}"
|
||||||
|
register: sg_facts
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
vpc_default_sg_id: "{{sg_facts.security_groups[0].group_id}}"
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Create Efs for testing
|
||||||
|
efs:
|
||||||
|
<<: *aws_connection_info
|
||||||
|
state: present
|
||||||
|
name: "{{ resource_prefix }}-test-efs"
|
||||||
|
tags:
|
||||||
|
Name: "{{ resource_prefix }}-test-tag"
|
||||||
|
Purpose: file-storage
|
||||||
|
targets:
|
||||||
|
- subnet_id: "{{testing_subnet_a.subnet.id}}"
|
||||||
|
- subnet_id: "{{testing_subnet_b.subnet.id}}"
|
||||||
|
register: created_efs
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get all EFS Facts
|
||||||
|
efs_facts:
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- (efs_result.ansible_facts.efs | length) >= 1
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get EFS by creation token
|
||||||
|
efs_facts:
|
||||||
|
name: "{{ resource_prefix }}-test-efs"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
efs_result_assertions:
|
||||||
|
- efs_result is not changed
|
||||||
|
- (efs_result.ansible_facts.efs | length) == 1
|
||||||
|
- efs_result.ansible_facts.efs[0].creation_token == "{{ resource_prefix }}-test-efs"
|
||||||
|
- efs_result.ansible_facts.efs[0].file_system_id == created_efs.efs.file_system_id
|
||||||
|
- efs_result.ansible_facts.efs[0].number_of_mount_targets == 2
|
||||||
|
- (efs_result.ansible_facts.efs[0].mount_targets | length) == 2
|
||||||
|
- efs_result.ansible_facts.efs[0].name == "{{ resource_prefix }}-test-tag"
|
||||||
|
- efs_result.ansible_facts.efs[0].tags.Name == "{{ resource_prefix }}-test-tag"
|
||||||
|
- efs_result.ansible_facts.efs[0].tags.Purpose == "file-storage"
|
||||||
|
- efs_result.ansible_facts.efs[0].encrypted == false
|
||||||
|
- efs_result.ansible_facts.efs[0].life_cycle_state == "available"
|
||||||
|
- efs_result.ansible_facts.efs[0].performance_mode == "generalPurpose"
|
||||||
|
- efs_result.ansible_facts.efs[0].mount_targets[0].security_groups[0] == vpc_default_sg_id
|
||||||
|
- efs_result.ansible_facts.efs[0].mount_targets[1].security_groups[0] == vpc_default_sg_id
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that: "{{efs_result_assertions}}"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get EFS by id
|
||||||
|
efs_facts:
|
||||||
|
id: "{{created_efs.efs.file_system_id}}"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that: "{{efs_result_assertions}}"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get EFS by tag
|
||||||
|
efs_facts:
|
||||||
|
tags:
|
||||||
|
Name: "{{ resource_prefix }}-test-tag"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that: "{{efs_result_assertions}}"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get EFS by target (subnet_id)
|
||||||
|
efs_facts:
|
||||||
|
targets:
|
||||||
|
- "{{testing_subnet_a.subnet.id}}"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that: "{{efs_result_assertions}}"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get EFS by target (security_group_id)
|
||||||
|
efs_facts:
|
||||||
|
targets:
|
||||||
|
- "{{vpc_default_sg_id}}"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that: "{{efs_result_assertions}}"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Get EFS by tag and target
|
||||||
|
efs_facts:
|
||||||
|
tags:
|
||||||
|
Name: "{{ resource_prefix }}-test-tag"
|
||||||
|
targets:
|
||||||
|
- "{{testing_subnet_a.subnet.id}}"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that: "{{efs_result_assertions}}"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
- name: Query unknown EFS by tag
|
||||||
|
efs_facts:
|
||||||
|
tags:
|
||||||
|
Name: "{{ resource_prefix }}-unknown"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- efs_result is not changed
|
||||||
|
- (efs_result.ansible_facts.efs | length) == 0
|
||||||
|
|
||||||
|
- name: Query unknown EFS by target
|
||||||
|
efs_facts:
|
||||||
|
targets:
|
||||||
|
- sg-00000000000
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: efs_result
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- efs_result is not changed
|
||||||
|
- (efs_result.ansible_facts.efs | length) == 0
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
always:
|
||||||
|
- name: Delete EFS used for tests
|
||||||
|
efs:
|
||||||
|
<<: *aws_connection_info
|
||||||
|
state: absent
|
||||||
|
name: "{{ resource_prefix }}-test-efs"
|
||||||
|
tags:
|
||||||
|
Name: "{{ resource_prefix }}-test-tag"
|
||||||
|
Purpose: file-storage
|
||||||
|
register: removed
|
||||||
|
until: removed is not failed
|
||||||
|
ignore_errors: yes
|
||||||
|
retries: 10
|
||||||
|
|
||||||
|
- name: Remove test subnet in zone A
|
||||||
|
ec2_vpc_subnet:
|
||||||
|
state: absent
|
||||||
|
vpc_id: "{{ testing_vpc.vpc.id }}"
|
||||||
|
cidr: 10.22.32.0/24
|
||||||
|
az: "{{ aws_region }}a"
|
||||||
|
resource_tags:
|
||||||
|
Name: "{{ resource_prefix }}-subnet-a"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: removed
|
||||||
|
until: removed is not failed
|
||||||
|
ignore_errors: yes
|
||||||
|
retries: 10
|
||||||
|
|
||||||
|
- name: Remove test subnet in zone B
|
||||||
|
ec2_vpc_subnet:
|
||||||
|
state: absent
|
||||||
|
vpc_id: "{{ testing_vpc.vpc.id }}"
|
||||||
|
cidr: 10.22.33.0/24
|
||||||
|
az: "{{ aws_region }}b"
|
||||||
|
resource_tags:
|
||||||
|
Name: "{{ resource_prefix }}-subnet-b"
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: removed
|
||||||
|
until: removed is not failed
|
||||||
|
ignore_errors: yes
|
||||||
|
retries: 10
|
||||||
|
|
||||||
|
- name: remove the VPC
|
||||||
|
ec2_vpc_net:
|
||||||
|
name: "{{ resource_prefix }}-vpc"
|
||||||
|
cidr_block: 10.22.32.0/23
|
||||||
|
state: absent
|
||||||
|
<<: *aws_connection_info
|
||||||
|
register: removed
|
||||||
|
until: removed is not failed
|
||||||
|
ignore_errors: yes
|
||||||
|
retries: 10
|
Loading…
Reference in a new issue