From 95aa86fcc3457968f647ebb74f5be49a17471b1c Mon Sep 17 00:00:00 2001 From: The Magician Date: Mon, 19 Aug 2019 12:53:30 -0700 Subject: [PATCH] Bug fixes for GCP modules (#60719) --- .../gcp_appengine_firewall_rule_info.py | 10 +-- .../cloud/google/gcp_bigquery_dataset_info.py | 10 +-- .../cloud/google/gcp_bigquery_table.py | 64 ++++++++++++++++++- .../cloud/google/gcp_bigquery_table_info.py | 33 +++++++--- .../google/gcp_cloudbuild_trigger_info.py | 10 +-- .../gcp_cloudfunctions_cloud_function_info.py | 10 +-- .../google/gcp_cloudscheduler_job_info.py | 10 +-- .../cloud/google/gcp_cloudtasks_queue_info.py | 10 +-- .../cloud/google/gcp_compute_address_info.py | 10 +-- .../google/gcp_compute_autoscaler_info.py | 10 +-- .../google/gcp_compute_backend_bucket_info.py | 10 +-- .../gcp_compute_backend_service_info.py | 10 +-- 12 files changed, 106 insertions(+), 91 deletions(-) diff --git a/lib/ansible/modules/cloud/google/gcp_appengine_firewall_rule_info.py b/lib/ansible/modules/cloud/google/gcp_appengine_firewall_rule_info.py index b24673b3df5..e2b6317a444 100644 --- a/lib/ansible/modules/cloud/google/gcp_appengine_firewall_rule_info.py +++ b/lib/ansible/modules/cloud/google/gcp_appengine_firewall_rule_info.py @@ -101,12 +101,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] - items = fetch_list(module, collection(module)) - if items.get('ingressRules'): - items = items.get('ingressRules') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -116,8 +111,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'appengine') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='ingressRules') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_bigquery_dataset_info.py b/lib/ansible/modules/cloud/google/gcp_bigquery_dataset_info.py index 9af87b929f1..3cc270f6a98 100644 --- a/lib/ansible/modules/cloud/google/gcp_bigquery_dataset_info.py +++ b/lib/ansible/modules/cloud/google/gcp_bigquery_dataset_info.py @@ -208,12 +208,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/bigquery'] - items = fetch_list(module, collection(module)) - if items.get('datasets'): - items = items.get('datasets') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -223,8 +218,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'bigquery') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='datasets') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_bigquery_table.py b/lib/ansible/modules/cloud/google/gcp_bigquery_table.py index 954bea4a17d..cdf262ba84a 100644 --- a/lib/ansible/modules/cloud/google/gcp_bigquery_table.py +++ b/lib/ansible/modules/cloud/google/gcp_bigquery_table.py @@ -69,6 +69,15 @@ options: - The ID of the the table. required: false type: str + clustering: + description: + - One or more fields on which data should be clustered. Only top-level, non-repeated, + simple-type fields are supported. When you cluster a table using multiple columns, + the order of columns you specify is important. The order of the specified columns + determines the sort order of the data. + required: false + type: list + version_added: 2.9 description: description: - A user-friendly description of the dataset. @@ -90,6 +99,13 @@ options: - Name of the table. required: false type: str + num_rows: + description: + - The number of rows of data in this table, excluding any data in the streaming + buffer. + required: false + type: int + version_added: 2.9 view: description: - The view definition. @@ -130,6 +146,16 @@ options: - Number of milliseconds for which to keep the storage for a partition. required: false type: int + field: + description: + - If not set, the table is partitioned by pseudo column, referenced via either + '_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE type. If + field is specified, the table is instead partitioned by this field. The + field must be a top-level TIMESTAMP or DATE field. Its mode must be NULLABLE + or REQUIRED. + required: false + type: str + version_added: 2.9 type: description: - The only type supported is DAY, which will generate one partition per day. @@ -485,6 +511,14 @@ tableReference: - The ID of the the table. returned: success type: str +clustering: + description: + - One or more fields on which data should be clustered. Only top-level, non-repeated, + simple-type fields are supported. When you cluster a table using multiple columns, + the order of columns you specify is important. The order of the specified columns + determines the sort order of the data. + returned: success + type: list creationTime: description: - The time when this dataset was created, in milliseconds since the epoch. @@ -543,6 +577,12 @@ numRows: buffer. returned: success type: int +requirePartitionFilter: + description: + - If set to true, queries over this table require a partition filter that can be + used for partition elimination to be specified. + returned: success + type: bool type: description: - Describes the table type. @@ -588,6 +628,14 @@ timePartitioning: - Number of milliseconds for which to keep the storage for a partition. returned: success type: int + field: + description: + - If not set, the table is partitioned by pseudo column, referenced via either + '_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE type. If field + is specified, the table is instead partitioned by this field. The field must + be a top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or REQUIRED. + returned: success + type: str type: description: - The only type supported is DAY, which will generate one partition per day. @@ -916,10 +964,12 @@ def main(): argument_spec=dict( state=dict(default='present', choices=['present', 'absent'], type='str'), table_reference=dict(type='dict', options=dict(dataset_id=dict(type='str'), project_id=dict(type='str'), table_id=dict(type='str'))), + clustering=dict(type='list', elements='str'), description=dict(type='str'), friendly_name=dict(type='str'), labels=dict(type='dict'), name=dict(type='str'), + num_rows=dict(type='int'), view=dict( type='dict', options=dict( @@ -929,7 +979,7 @@ def main(): ), ), ), - time_partitioning=dict(type='dict', options=dict(expiration_ms=dict(type='int'), type=dict(type='str'))), + time_partitioning=dict(type='dict', options=dict(expiration_ms=dict(type='int'), field=dict(type='str'), type=dict(type='str'))), schema=dict( type='dict', options=dict( @@ -1069,10 +1119,12 @@ def resource_to_request(module): request = { u'kind': 'bigquery#table', u'tableReference': TableTablereference(module.params.get('table_reference', {}), module).to_request(), + u'clustering': module.params.get('clustering'), u'description': module.params.get('description'), u'friendlyName': module.params.get('friendly_name'), u'labels': module.params.get('labels'), u'name': module.params.get('name'), + u'numRows': module.params.get('num_rows'), u'view': TableView(module.params.get('view', {}), module).to_request(), u'timePartitioning': TableTimepartitioning(module.params.get('time_partitioning', {}), module).to_request(), u'schema': TableSchema(module.params.get('schema', {}), module).to_request(), @@ -1145,6 +1197,7 @@ def is_different(module, response): def response_to_hash(module, response): return { u'tableReference': TableTablereference(response.get(u'tableReference', {}), module).from_response(), + u'clustering': response.get(u'clustering'), u'creationTime': response.get(u'creationTime'), u'description': response.get(u'description'), u'friendlyName': response.get(u'friendlyName'), @@ -1156,6 +1209,7 @@ def response_to_hash(module, response): u'numBytes': response.get(u'numBytes'), u'numLongTermBytes': response.get(u'numLongTermBytes'), u'numRows': response.get(u'numRows'), + u'requirePartitionFilter': response.get(u'requirePartitionFilter'), u'type': response.get(u'type'), u'view': TableView(response.get(u'view', {}), module).from_response(), u'timePartitioning': TableTimepartitioning(response.get(u'timePartitioning', {}), module).from_response(), @@ -1251,10 +1305,14 @@ class TableTimepartitioning(object): self.request = {} def to_request(self): - return remove_nones_from_dict({u'expirationMs': self.request.get('expiration_ms'), u'type': self.request.get('type')}) + return remove_nones_from_dict( + {u'expirationMs': self.request.get('expiration_ms'), u'field': self.request.get('field'), u'type': self.request.get('type')} + ) def from_response(self): - return remove_nones_from_dict({u'expirationMs': self.request.get(u'expirationMs'), u'type': self.request.get(u'type')}) + return remove_nones_from_dict( + {u'expirationMs': self.request.get(u'expirationMs'), u'field': self.request.get(u'field'), u'type': self.request.get(u'type')} + ) class TableStreamingbuffer(object): diff --git a/lib/ansible/modules/cloud/google/gcp_bigquery_table_info.py b/lib/ansible/modules/cloud/google/gcp_bigquery_table_info.py index 732c592cb57..1491deeabcd 100644 --- a/lib/ansible/modules/cloud/google/gcp_bigquery_table_info.py +++ b/lib/ansible/modules/cloud/google/gcp_bigquery_table_info.py @@ -86,6 +86,14 @@ resources: - The ID of the the table. returned: success type: str + clustering: + description: + - One or more fields on which data should be clustered. Only top-level, non-repeated, + simple-type fields are supported. When you cluster a table using multiple + columns, the order of columns you specify is important. The order of the specified + columns determines the sort order of the data. + returned: success + type: list creationTime: description: - The time when this dataset was created, in milliseconds since the epoch. @@ -144,6 +152,12 @@ resources: buffer. returned: success type: int + requirePartitionFilter: + description: + - If set to true, queries over this table require a partition filter that can + be used for partition elimination to be specified. + returned: success + type: bool type: description: - Describes the table type. @@ -189,6 +203,15 @@ resources: - Number of milliseconds for which to keep the storage for a partition. returned: success type: int + field: + description: + - If not set, the table is partitioned by pseudo column, referenced via + either '_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE + type. If field is specified, the table is instead partitioned by this + field. The field must be a top-level TIMESTAMP or DATE field. Its mode + must be NULLABLE or REQUIRED. + returned: success + type: str type: description: - The only type supported is DAY, which will generate one partition per @@ -523,12 +546,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/bigquery'] - items = fetch_list(module, collection(module)) - if items.get('tables'): - items = items.get('tables') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -538,8 +556,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'bigquery') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='tables') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_cloudbuild_trigger_info.py b/lib/ansible/modules/cloud/google/gcp_cloudbuild_trigger_info.py index dabacbdd231..ec581c6f3e7 100644 --- a/lib/ansible/modules/cloud/google/gcp_cloudbuild_trigger_info.py +++ b/lib/ansible/modules/cloud/google/gcp_cloudbuild_trigger_info.py @@ -324,12 +324,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] - items = fetch_list(module, collection(module)) - if items.get('triggers'): - items = items.get('triggers') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -339,8 +334,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'cloudbuild') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='triggers') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_cloudfunctions_cloud_function_info.py b/lib/ansible/modules/cloud/google/gcp_cloudfunctions_cloud_function_info.py index 8cfc9522753..34aa2c1b4a3 100644 --- a/lib/ansible/modules/cloud/google/gcp_cloudfunctions_cloud_function_info.py +++ b/lib/ansible/modules/cloud/google/gcp_cloudfunctions_cloud_function_info.py @@ -224,12 +224,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] - items = fetch_list(module, collection(module)) - if items.get('functions'): - items = items.get('functions') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -239,8 +234,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'cloudfunctions') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='functions') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_cloudscheduler_job_info.py b/lib/ansible/modules/cloud/google/gcp_cloudscheduler_job_info.py index 831575188dc..7aee6c8d6d3 100644 --- a/lib/ansible/modules/cloud/google/gcp_cloudscheduler_job_info.py +++ b/lib/ansible/modules/cloud/google/gcp_cloudscheduler_job_info.py @@ -311,12 +311,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] - items = fetch_list(module, collection(module)) - if items.get('jobs'): - items = items.get('jobs') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -326,8 +321,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'cloudscheduler') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='jobs') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_cloudtasks_queue_info.py b/lib/ansible/modules/cloud/google/gcp_cloudtasks_queue_info.py index b84e6a67630..0598c87927d 100644 --- a/lib/ansible/modules/cloud/google/gcp_cloudtasks_queue_info.py +++ b/lib/ansible/modules/cloud/google/gcp_cloudtasks_queue_info.py @@ -216,12 +216,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform'] - items = fetch_list(module, collection(module)) - if items.get('queues'): - items = items.get('queues') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module))} module.exit_json(**return_value) @@ -231,8 +226,7 @@ def collection(module): def fetch_list(module, link): auth = GcpSession(module, 'cloudtasks') - response = auth.get(link) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='queues') def return_if_object(module, response): diff --git a/lib/ansible/modules/cloud/google/gcp_compute_address_info.py b/lib/ansible/modules/cloud/google/gcp_compute_address_info.py index 93e3ee89df7..a7791eaa36d 100644 --- a/lib/ansible/modules/cloud/google/gcp_compute_address_info.py +++ b/lib/ansible/modules/cloud/google/gcp_compute_address_info.py @@ -159,12 +159,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/compute'] - items = fetch_list(module, collection(module), query_options(module.params['filters'])) - if items.get('items'): - items = items.get('items') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))} module.exit_json(**return_value) @@ -174,8 +169,7 @@ def collection(module): def fetch_list(module, link, query): auth = GcpSession(module, 'compute') - response = auth.get(link, params={'filter': query}) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='items', params={'filter': query}) def query_options(filters): diff --git a/lib/ansible/modules/cloud/google/gcp_compute_autoscaler_info.py b/lib/ansible/modules/cloud/google/gcp_compute_autoscaler_info.py index 18481849b9a..8c957f8b09f 100644 --- a/lib/ansible/modules/cloud/google/gcp_compute_autoscaler_info.py +++ b/lib/ansible/modules/cloud/google/gcp_compute_autoscaler_info.py @@ -224,12 +224,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/compute'] - items = fetch_list(module, collection(module), query_options(module.params['filters'])) - if items.get('items'): - items = items.get('items') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))} module.exit_json(**return_value) @@ -239,8 +234,7 @@ def collection(module): def fetch_list(module, link, query): auth = GcpSession(module, 'compute') - response = auth.get(link, params={'filter': query}) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='items', params={'filter': query}) def query_options(filters): diff --git a/lib/ansible/modules/cloud/google/gcp_compute_backend_bucket_info.py b/lib/ansible/modules/cloud/google/gcp_compute_backend_bucket_info.py index c7d214b8ec4..2d6c62a265e 100644 --- a/lib/ansible/modules/cloud/google/gcp_compute_backend_bucket_info.py +++ b/lib/ansible/modules/cloud/google/gcp_compute_backend_bucket_info.py @@ -142,12 +142,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/compute'] - items = fetch_list(module, collection(module), query_options(module.params['filters'])) - if items.get('items'): - items = items.get('items') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))} module.exit_json(**return_value) @@ -157,8 +152,7 @@ def collection(module): def fetch_list(module, link, query): auth = GcpSession(module, 'compute') - response = auth.get(link, params={'filter': query}) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='items', params={'filter': query}) def query_options(filters): diff --git a/lib/ansible/modules/cloud/google/gcp_compute_backend_service_info.py b/lib/ansible/modules/cloud/google/gcp_compute_backend_service_info.py index f1535151c84..5d529126bf5 100644 --- a/lib/ansible/modules/cloud/google/gcp_compute_backend_service_info.py +++ b/lib/ansible/modules/cloud/google/gcp_compute_backend_service_info.py @@ -379,12 +379,7 @@ def main(): if not module.params['scopes']: module.params['scopes'] = ['https://www.googleapis.com/auth/compute'] - items = fetch_list(module, collection(module), query_options(module.params['filters'])) - if items.get('items'): - items = items.get('items') - else: - items = [] - return_value = {'resources': items} + return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))} module.exit_json(**return_value) @@ -394,8 +389,7 @@ def collection(module): def fetch_list(module, link, query): auth = GcpSession(module, 'compute') - response = auth.get(link, params={'filter': query}) - return return_if_object(module, response) + return auth.list(link, return_if_object, array_name='items', params={'filter': query}) def query_options(filters):