Allow passing through of (almost) all params available … (#58118)

* Allow passing through of (almost) all params available on boto methods in aws_api_gateway

* Linting and docs fixes

* Refactored method signature of create_deployment() to use keyword args instead of named args

* Updated version_added flags to 2.10

* Cleanup and improve aws_api__gateway integration test play. Also included new params into test.

* Fixed RETURN docs and some ttests

* Completed RETURN docs and made integration tests match

* Fixed variable names in test and YAML syntax in docs

* Comment out critical sections of integration test

* Fixed update test after figuring out what the error message means. Also updated error message to be more descriptive.

* Fixed test assertion

* Update docs and make tests reflect that endpoint type wont be changed on updates

* Syntax fix

* Add changelog fragment

* Improve aws_api_gateway docs, fix typos.

* Quote doc lines with colon

changelogs/fragments/58118-aws_api_gateway-params.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - Allow all params that boto support in aws_api_gateway module

lib/ansible/modules/cloud/amazon/aws_api_gateway.py

@@ -37,8 +37,7 @@ options:
       - The ID of the API you want to manage.
     type: str
   state:
-    description:
+    description: Create or delete API Gateway.
-      - NOT IMPLEMENTED Create or delete API - currently we always create.
     default: present
     choices: [ 'present', 'absent' ]
     type: str
@@ -69,6 +68,49 @@ options:
         AWS console.
     default: Automatic deployment by Ansible.
     type: str
+  cache_enabled:
+    description:
+      - Enable API GW caching of backend responses. Defaults to false.
+    type: bool
+    default: false
+    version_added: '2.10'
+  cache_size:
+    description:
+      - Size in GB of the API GW cache, becomes effective when cache_enabled is true.
+    choices: ['0.5', '1.6', '6.1', '13.5', '28.4', '58.2', '118', '237']
+    type: str
+    default: '0.5'
+    version_added: '2.10'
+  stage_variables:
+    description:
+      - ENV variables for the stage. Define a dict of key values pairs for variables.
+    type: dict
+    version_added: '2.10'
+  stage_canary_settings:
+    description:
+      - Canary settings for the deployment of the stage.
+      - 'Dict with following settings:'
+      - 'percentTraffic: The percent (0-100) of traffic diverted to a canary deployment.'
+      - 'deploymentId: The ID of the canary deployment.'
+      - 'stageVariableOverrides: Stage variables overridden for a canary release deployment.'
+      - 'useStageCache: A Boolean flag to indicate whether the canary deployment uses the stage cache or not.'
+      - See docs U(https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/apigateway.html#APIGateway.Client.create_stage)
+    type: dict
+    version_added: '2.10'
+  tracing_enabled:
+    description:
+      - Specifies whether active tracing with X-ray is enabled for the API GW stage.
+    type: bool
+    version_added: '2.10'
+  endpoint_type:
+    description:
+      - Type of endpoint configuration, use C(EDGE) for an edge optimized API endpoint,
+      - C(REGIONAL) for just a regional deploy or PRIVATE for a private API.
+      - This will flag will only be used when creating a new API Gateway setup, not for updates.
+    choices: ['EDGE', 'REGIONAL', 'PRIVATE']
+    type: str
+    default: EDGE
+    version_added: '2.10'
 author:
     - 'Michael De La Rue (@mikedlr)'
 extends_documentation_fragment:
@@ -83,35 +125,57 @@ notes:
 '''
 
 EXAMPLES = '''
-# Update API resources for development
+- name: Setup AWS API Gateway setup on AWS and deploy API definition
-- name: update API
   aws_api_gateway:
-    api_id: 'abc123321cba'
-    state: present
-    swagger_file: my_api.yml
-
-# update definitions and deploy API to production
-- name: deploy API
-  aws_api_gateway:
-    api_id: 'abc123321cba'
-    state: present
     swagger_file: my_api.yml
     stage: production
+    cache_enabled: true
+    cache_size: '1.6'
+    tracing_enabled: true
+    endpoint_type: EDGE
+    state: present
+
+- name: Update API definition to deploy new version
+  aws_api_gateway:
+    api_id: 'abc123321cba'
+    swagger_file: my_api.yml
     deploy_desc: Make auth fix available.
+    cache_enabled: true
+    cache_size: '1.6'
+    endpoint_type: EDGE
+    state: present
+
+- name: Update API definitions and settings and deploy as canary
+  aws_api_gateway:
+    api_id: 'abc123321cba'
+    swagger_file: my_api.yml
+    cache_enabled: true
+    cache_size: '6.1'
+    canary_settings: { percentTraffic: 50.0, deploymentId: '123', useStageCache: True }
+    state: present
 '''
 
 RETURN = '''
-output:
+api_id:
-  description: the data returned by put_restapi in boto3
+    description: API id of the API endpoint created
-  returned: success
+    returned: success
-  type: dict
+    type: str
-  sample:
+    sample: '0ln4zq7p86'
-    'data':
+configure_response:
-      {
+    description: AWS response from the API configure call
-          "id": "abc123321cba",
+    returned: success
-          "name": "MY REST API",
+    type: dict
-          "createdDate": 1484233401
+    sample: { api_key_source: "HEADER", created_at: "2020-01-01T11:37:59+00:00", id: "0ln4zq7p86" }
-      }
+deploy_response:
+    description: AWS response from the API deploy call
+    returned: success
+    type: dict
+    sample: { created_date: "2020-01-01T11:36:59+00:00", id: "rptv4b", description: "Automatic deployment by Ansible." }
+resource_actions:
+    description: Actions performed against AWS API
+    returned: always
+    type: list
+    sample: ["apigateway:CreateRestApi", "apigateway:CreateDeployment", "apigateway:PutRestApi"]
 '''
 
 import json
@@ -136,6 +200,12 @@ def main():
         swagger_text=dict(type='str', default=None),
         stage=dict(type='str', default=None),
         deploy_desc=dict(type='str', default="Automatic deployment by Ansible."),
+        cache_enabled=dict(type='bool', default=False),
+        cache_size=dict(type='str', default='0.5', choices=['0.5', '1.6', '6.1', '13.5', '28.4', '58.2', '118', '237']),
+        stage_variables=dict(type='dict', default={}),
+        stage_canary_settings=dict(type='dict', default={}),
+        tracing_enabled=dict(type='bool', default=False),
+        endpoint_type=dict(type='str', default='EDGE', choices=['EDGE', 'REGIONAL', 'PRIVATE'])
     )
 
     mutually_exclusive = [['swagger_file', 'swagger_dict', 'swagger_text']]  # noqa: F841
@@ -151,8 +221,7 @@ def main():
     swagger_file = module.params.get('swagger_file')
     swagger_dict = module.params.get('swagger_dict')
     swagger_text = module.params.get('swagger_text')
-    stage = module.params.get('stage')
+    endpoint_type = module.params.get('endpoint_type')
-    deploy_desc = module.params.get('deploy_desc')
 
     client = module.client('apigateway')
 
@@ -163,12 +232,10 @@ def main():
 
     if state == "present":
         if api_id is None:
-            api_id = create_empty_api(module, client)
+            api_id = create_empty_api(module, client, endpoint_type)
         api_data = get_api_definitions(module, swagger_file=swagger_file,
                                        swagger_dict=swagger_dict, swagger_text=swagger_text)
-        conf_res, dep_res = ensure_api_in_correct_state(module, client, api_id=api_id,
+        conf_res, dep_res = ensure_api_in_correct_state(module, client, api_id, api_data)
-                                                        api_data=api_data, stage=stage,
-                                                        deploy_desc=deploy_desc)
     if state == "absent":
         del_res = delete_rest_api(module, client, api_id)
 
@@ -199,19 +266,19 @@ def get_api_definitions(module, swagger_file=None, swagger_dict=None, swagger_te
         apidata = swagger_text
 
     if apidata is None:
-        module.fail_json(msg='module error - failed to get API data')
+        module.fail_json(msg='module error - no swagger info provided')
     return apidata
 
 
-def create_empty_api(module, client):
+def create_empty_api(module, client, endpoint_type):
     """
-    creates a new empty API ready to be configured.  The description is
+    creates a new empty API ready to be configured. The description is
     temporarily set to show the API as incomplete but should be
     updated when the API is configured.
     """
     desc = "Incomplete API creation by ansible aws_api_gateway module"
     try:
-        awsret = create_api(client, name="ansible-temp-api", description=desc)
+        awsret = create_api(client, name="ansible-temp-api", description=desc, endpoint_type=endpoint_type)
     except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
         module.fail_json_aws(e, msg="creating API")
     return awsret["id"]
@@ -219,19 +286,16 @@ def create_empty_api(module, client):
 
 def delete_rest_api(module, client, api_id):
     """
-    creates a new empty API ready to be configured.  The description is
+    Deletes entire REST API setup
-    temporarily set to show the API as incomplete but should be
-    updated when the API is configured.
     """
     try:
-        delete_response = delete_api(client, api_id=api_id)
+        delete_response = delete_api(client, api_id)
     except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
         module.fail_json_aws(e, msg="deleting API {0}".format(api_id))
     return delete_response
 
 
-def ensure_api_in_correct_state(module, client, api_id=None, api_data=None, stage=None,
+def ensure_api_in_correct_state(module, client, api_id, api_data):
-                                deploy_desc=None):
     """Make sure that we have the API configured and deployed as instructed.
 
     This function first configures the API correctly uploading the
@@ -243,16 +307,16 @@ def ensure_api_in_correct_state(module, client, api_id=None, api_data=None, stag
 
     configure_response = None
     try:
-        configure_response = configure_api(client, api_data=api_data, api_id=api_id)
+        configure_response = configure_api(client, api_id, api_data=api_data)
     except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
         module.fail_json_aws(e, msg="configuring API {0}".format(api_id))
 
     deploy_response = None
 
+    stage = module.params.get('stage')
     if stage:
         try:
-            deploy_response = create_deployment(client, api_id=api_id, stage=stage,
+            deploy_response = create_deployment(client, api_id, **module.params)
-                                                description=deploy_desc)
         except (botocore.exceptions.ClientError, botocore.exceptions.EndpointConnectionError) as e:
             msg = "deploying api {0} to stage {1}".format(api_id, stage)
             module.fail_json_aws(e, msg)
@@ -264,24 +328,47 @@ retry_params = {"tries": 10, "delay": 5, "backoff": 1.2}
 
 
 @AWSRetry.backoff(**retry_params)
-def create_api(client, name=None, description=None):
+def create_api(client, name=None, description=None, endpoint_type=None):
-    return client.create_rest_api(name="ansible-temp-api", description=description)
+    return client.create_rest_api(name="ansible-temp-api", description=description, endpointConfiguration={'types': [endpoint_type]})
 
 
 @AWSRetry.backoff(**retry_params)
-def delete_api(client, api_id=None):
+def delete_api(client, api_id):
     return client.delete_rest_api(restApiId=api_id)
 
 
 @AWSRetry.backoff(**retry_params)
-def configure_api(client, api_data=None, api_id=None, mode="overwrite"):
+def configure_api(client, api_id, api_data=None, mode="overwrite"):
-    return client.put_rest_api(body=api_data, restApiId=api_id, mode=mode)
+    return client.put_rest_api(restApiId=api_id, mode=mode, body=api_data)
 
 
 @AWSRetry.backoff(**retry_params)
-def create_deployment(client, api_id=None, stage=None, description=None):
+def create_deployment(client, rest_api_id, **params):
-    # we can also get None as an argument so we don't do this as a default
+    canary_settings = params.get('stage_canary_settings')
-    return client.create_deployment(restApiId=api_id, stageName=stage, description=description)
+
+    if canary_settings and len(canary_settings) > 0:
+        result = client.create_deployment(
+            restApiId=rest_api_id,
+            stageName=params.get('stage'),
+            description=params.get('deploy_desc'),
+            cacheClusterEnabled=params.get('cache_enabled'),
+            cacheClusterSize=params.get('cache_size'),
+            variables=params.get('stage_variables'),
+            canarySettings=canary_settings,
+            tracingEnabled=params.get('tracing_enabled')
+        )
+    else:
+        result = client.create_deployment(
+            restApiId=rest_api_id,
+            stageName=params.get('stage'),
+            description=params.get('deploy_desc'),
+            cacheClusterEnabled=params.get('cache_enabled'),
+            cacheClusterSize=params.get('cache_size'),
+            variables=params.get('stage_variables'),
+            tracingEnabled=params.get('tracing_enabled')
+        )
+
+    return result
 
 
 if __name__ == '__main__':

test/integration/targets/aws_api_gateway/tasks/main.yml

@@ -1,6 +1,7 @@
 - block:
 
-    # ============================================================
+    # ====================== testing failure cases: ==================================
+
     - name: test with no parameters
       aws_api_gateway:
       register: result
@@ -12,7 +13,6 @@
            - 'result.failed'
            - 'result.msg.startswith("The aws_api_gateway module requires a region")'
 
-    # ============================================================
     - name: test with minimal parameters but no region
       aws_api_gateway:
         api_id: 'fake-api-doesnt-exist'
@@ -25,11 +25,10 @@
            - 'result.failed'
            - 'result.msg.startswith("The aws_api_gateway module requires a region")'
 
-    # ============================================================
+    - name: test for disallowing multiple swagger sources
-    - name: test disallow multiple swagger sources
       aws_api_gateway:
         api_id: 'fake-api-doesnt-exist'
-        region: 'fake_region'
+        region: '{{ec2_region}}'
         swagger_file: foo.yml
         swagger_text: "this is not really an API"
       register: result
@@ -41,40 +40,20 @@
            - 'result.failed'
            - 'result.msg.startswith("parameters are mutually exclusive")'
 
-    # This fails with
 
-    # msg": "There is an issue in the code of the module. You must
+    # ====================== regular testing: ===================================
-    # specify either both, resource or client to the conn_type
-    # parameter in the boto3_conn function call"
 
-    # even though the call appears to include conn_type='client'
+    - name: build API file
-
-    # # ============================================================
-    # - name: test invalid region parameter
-    #   aws_api_gateway:
-    #     api_id: 'fake-api-doesnt-exist'
-    #     region: 'asdf querty 1234'
-    #   register: result
-    #   ignore_errors: true
-
-    # - name: assert invalid region parameter
-    #   assert:
-    #     that:
-    #        - 'result.failed'
-    #        - 'result.msg.startswith("Region asdf querty 1234 does not seem to be available ")'
-
-    # ============================================================
-
-    - name: build API file 
       template:
         src: minimal-swagger-api.yml.j2
         dest: "{{output_dir}}/minimal-swagger-api.yml"
-      tags: new_api,api,api_file
 
     - name: deploy new API
-      aws_api_gateway: 
+      aws_api_gateway:
         api_file: "{{output_dir}}/minimal-swagger-api.yml"
         stage: "minimal"
+        endpoint_type: 'REGIONAL'
+        state: present
         region: '{{ec2_region}}'
         aws_access_key: '{{ec2_access_key}}'
         aws_secret_key: '{{ec2_secret_key}}'
@@ -84,36 +63,61 @@
     - name: assert deploy new API worked
       assert:
         that:
-           - 'create_result.changed == True'
+          - 'create_result.changed == True'
-           - '"api_id" in create_result'
+          - 'create_result.failed == False'
-#           - '"created_response.created_date" in create_result'
+          - 'create_result.deploy_response.description == "Automatic deployment by Ansible."'
-#           - '"deploy_response.created_date" in create_result'
+          - 'create_result.configure_response.id == create_result.api_id'
+          - '"apigateway:CreateRestApi" in create_result.resource_actions'
+          - 'create_result.configure_response.endpoint_configuration.types.0 == "REGIONAL"'
 
-    - name: check API works
+    - name: check if API endpoint works
       uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/minimal"
       register: uri_result
 
     - name: assert API works success
       assert:
         that:
-           - 'uri_result'
+          - 'uri_result.status == 200'
 
-    - name: check nonexistent endpoints cause errors
+    - name: check if nonexistent endpoint causes error
       uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/nominal"
       register: bad_uri_result
       ignore_errors: true
 
-    - name: assert 
+    - name: assert
       assert:
         that:
           - bad_uri_result is failed
 
-    # ============================================================
+    - name: Update API to test params effect
+      aws_api_gateway:
+        api_id: '{{create_result.api_id}}'
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        cache_enabled: true
+        cache_size: '1.6'
+        tracing_enabled: true
+        state: present
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: update_result
+
+    - name: assert update result
+      assert:
+        that:
+          - 'update_result.changed == True'
+          - 'update_result.failed == False'
+          - '"apigateway:PutRestApi" in update_result.resource_actions'
+
+    # ==== additional create/delete tests ====
 
     - name: deploy first API
       aws_api_gateway:
         api_file: "{{output_dir}}/minimal-swagger-api.yml"
         stage: "minimal"
+        cache_enabled: false
+        state: present
         region: '{{ec2_region}}'
         aws_access_key: '{{ec2_access_key}}'
         aws_secret_key: '{{ec2_secret_key}}'
@@ -124,6 +128,7 @@
       aws_api_gateway:
         api_file: "{{output_dir}}/minimal-swagger-api.yml"
         stage: "minimal"
+        state: present
         region: '{{ec2_region}}'
         aws_access_key: '{{ec2_access_key}}'
         aws_secret_key: '{{ec2_secret_key}}'
@@ -133,12 +138,11 @@
     - name: assert both APIs deployed successfully
       assert:
         that:
-           - 'create_result_1.changed == True'
+          - 'create_result_1.changed == True'
-           - 'create_result_2.changed == True'
+          - 'create_result_2.changed == True'
-           - '"api_id" in create_result_1'
+          - '"api_id" in create_result_1'
-           - '"api_id" in create_result_1'
+          - '"api_id" in create_result_1'
-#           - '"created_response.created_date" in create_result'
+          - 'create_result_1.configure_response.endpoint_configuration.types.0 == "EDGE"'
-#           - '"deploy_response.created_date" in create_result'
 
     - name: destroy first API
       aws_api_gateway:
@@ -165,18 +169,39 @@
         that:
            - 'destroy_result_1.changed == True'
            - 'destroy_result_2.changed == True'
-#           - '"created_response.created_date" in create_result'
+           - '"apigateway:DeleteRestApi" in destroy_result_1.resource_actions'
-#           - '"deploy_response.created_date" in create_result'
+           - '"apigateway:DeleteRestApi" in destroy_result_2.resource_actions'
+
+    # ================= end testing ====================================
 
   always:
 
-    # ============================================================
+    - name: Ensure cleanup of API deploy
-    - name: test state=absent (expect changed=false)
       aws_api_gateway:
         state: absent
-        api_id: '{{create_result.api_id}}'        
+        api_id: '{{create_result.api_id}}'
         ec2_region: '{{ec2_region}}'
         aws_access_key: '{{ec2_access_key}}'
         aws_secret_key: '{{ec2_secret_key}}'
         security_token: '{{security_token}}'
-      register: destroy_result
+      ignore_errors: true
+
+    - name: Ensure cleanup of API deploy 1
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result_1.api_id}}'
+        ec2_region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      ignore_errors: true
+
+    - name: Ensure cleanup of API deploy 2
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result_2.api_id}}'
+        ec2_region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      ignore_errors: true