New module: gcp_bigquery_table_facts (#50679)
This commit is contained in:
Alex Stephen
ansibot
parent
13a2a8b5f6
commit
1d747649e6
2 changed files with 607 additions and 8 deletions
|
|
@ -19,6 +19,7 @@ except ImportError:
|
|||
from ansible.module_utils.basic import AnsibleModule, env_fallback
|
||||
from ansible.module_utils.six import string_types
|
||||
from ansible.module_utils._text import to_text
|
||||
import ast
|
||||
import os
|
||||
|
||||
|
||||
|
|
@ -60,10 +61,18 @@ def replace_resource_dict(item, value):
|
|||
else:
|
||||
if not item:
|
||||
return item
|
||||
return item.get(value)
|
||||
if isinstance(item, dict):
|
||||
return item.get(value)
|
||||
|
||||
# Item could be a string or a string representing a dictionary.
|
||||
try:
|
||||
new_item = ast.literal_eval(item)
|
||||
return replace_resource_dict(new_item, value)
|
||||
except ValueError:
|
||||
return new_item
|
||||
|
||||
|
||||
# Handles all authentication and HTTP sessions for GCP API calls.
|
||||
# Handles all authentation and HTTP sessions for GCP API calls.
|
||||
class GcpSession(object):
|
||||
def __init__(self, module, product):
|
||||
self.module = module
|
||||
|
|
@ -77,9 +86,25 @@ class GcpSession(object):
|
|||
except getattr(requests.exceptions, 'RequestException') as inst:
|
||||
self.module.fail_json(msg=inst.message)
|
||||
|
||||
def post(self, url, body=None):
|
||||
def post(self, url, body=None, headers=None, **kwargs):
|
||||
if headers:
|
||||
headers = self.merge_dictionaries(headers, self._headers())
|
||||
else:
|
||||
headers = self._headers()
|
||||
|
||||
try:
|
||||
return self.session().post(url, json=body, headers=self._headers())
|
||||
return self.session().post(url, json=body, headers=headers)
|
||||
except getattr(requests.exceptions, 'RequestException') as inst:
|
||||
self.module.fail_json(msg=inst.message)
|
||||
|
||||
def post_contents(self, url, file_contents=None, headers=None, **kwargs):
|
||||
if headers:
|
||||
headers = self.merge_dictionaries(headers, self._headers())
|
||||
else:
|
||||
headers = self._headers()
|
||||
|
||||
try:
|
||||
return self.session().post(url, data=file_contents, headers=headers)
|
||||
except getattr(requests.exceptions, 'RequestException') as inst:
|
||||
self.module.fail_json(msg=inst.message)
|
||||
|
||||
|
|
@ -103,7 +128,8 @@ class GcpSession(object):
|
|||
self.module.fail_json(msg=inst.message)
|
||||
|
||||
def session(self):
|
||||
return AuthorizedSession(self._credentials())
|
||||
return AuthorizedSession(
|
||||
self._credentials())
|
||||
|
||||
def _validate(self):
|
||||
if not HAS_REQUESTS:
|
||||
|
|
@ -114,12 +140,12 @@ class GcpSession(object):
|
|||
|
||||
if self.module.params.get('service_account_email') is not None and self.module.params['auth_kind'] != 'machineaccount':
|
||||
self.module.fail_json(
|
||||
msg="Service Account Email only works with Machine Account-based authentication"
|
||||
msg="Service Acccount Email only works with Machine Account-based authentication"
|
||||
)
|
||||
|
||||
if self.module.params.get('service_account_file') is not None and self.module.params['auth_kind'] != 'serviceaccount':
|
||||
self.module.fail_json(
|
||||
msg="Service Account File only works with Service Account-based authentication"
|
||||
msg="Service Acccount File only works with Service Account-based authentication"
|
||||
)
|
||||
|
||||
def _credentials(self):
|
||||
|
|
@ -134,13 +160,18 @@ class GcpSession(object):
|
|||
return google.auth.compute_engine.Credentials(
|
||||
self.module.params['service_account_email'])
|
||||
else:
|
||||
self.module.fail_json(msg="Credential type '%s' not implemented" % cred_type)
|
||||
self.module.fail_json(msg="Credential type '%s' not implmented" % cred_type)
|
||||
|
||||
def _headers(self):
|
||||
return {
|
||||
'User-Agent': "Google-Ansible-MM-{0}".format(self.product)
|
||||
}
|
||||
|
||||
def _merge_dictionaries(self, a, b):
|
||||
new = a.copy()
|
||||
new.update(b)
|
||||
return new
|
||||
|
||||
|
||||
class GcpModule(AnsibleModule):
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
|
|
|||
568
lib/ansible/modules/cloud/google/gcp_bigquery_table_facts.py
Normal file
568
lib/ansible/modules/cloud/google/gcp_bigquery_table_facts.py
Normal file
|
|
@ -0,0 +1,568 @@
|
|||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2017 Google
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
# ----------------------------------------------------------------------------
|
||||
#
|
||||
# *** AUTO GENERATED CODE *** AUTO GENERATED CODE ***
|
||||
#
|
||||
# ----------------------------------------------------------------------------
|
||||
#
|
||||
# This file is automatically generated by Magic Modules and manual
|
||||
# changes will be clobbered when the file is regenerated.
|
||||
#
|
||||
# Please read more about how to change this file at
|
||||
# https://www.github.com/GoogleCloudPlatform/magic-modules
|
||||
#
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
__metaclass__ = type
|
||||
|
||||
################################################################################
|
||||
# Documentation
|
||||
################################################################################
|
||||
|
||||
ANSIBLE_METADATA = {'metadata_version': '1.1',
|
||||
'status': ["preview"],
|
||||
'supported_by': 'community'}
|
||||
|
||||
DOCUMENTATION = '''
|
||||
---
|
||||
module: gcp_bigquery_table_facts
|
||||
description:
|
||||
- Gather facts for GCP Table
|
||||
short_description: Gather facts for GCP Table
|
||||
version_added: 2.8
|
||||
author: Google Inc. (@googlecloudplatform)
|
||||
requirements:
|
||||
- python >= 2.6
|
||||
- requests >= 2.18.4
|
||||
- google-auth >= 1.3.0
|
||||
options:
|
||||
dataset:
|
||||
description:
|
||||
- Name of the dataset.
|
||||
required: false
|
||||
extends_documentation_fragment: gcp
|
||||
'''
|
||||
|
||||
EXAMPLES = '''
|
||||
- name: a table facts
|
||||
gcp_bigquery_table_facts:
|
||||
dataset: example_dataset
|
||||
project: test_project
|
||||
auth_kind: serviceaccount
|
||||
service_account_file: "/tmp/auth.pem"
|
||||
'''
|
||||
|
||||
RETURN = '''
|
||||
items:
|
||||
description: List of items
|
||||
returned: always
|
||||
type: complex
|
||||
contains:
|
||||
tableReference:
|
||||
description:
|
||||
- Reference describing the ID of this table.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
datasetId:
|
||||
description:
|
||||
- The ID of the dataset containing this table.
|
||||
returned: success
|
||||
type: str
|
||||
projectId:
|
||||
description:
|
||||
- The ID of the project containing this table.
|
||||
returned: success
|
||||
type: str
|
||||
tableId:
|
||||
description:
|
||||
- The ID of the the table.
|
||||
returned: success
|
||||
type: str
|
||||
creationTime:
|
||||
description:
|
||||
- The time when this dataset was created, in milliseconds since the epoch.
|
||||
returned: success
|
||||
type: int
|
||||
description:
|
||||
description:
|
||||
- A user-friendly description of the dataset.
|
||||
returned: success
|
||||
type: str
|
||||
friendlyName:
|
||||
description:
|
||||
- A descriptive name for this table.
|
||||
returned: success
|
||||
type: str
|
||||
id:
|
||||
description:
|
||||
- An opaque ID uniquely identifying the table.
|
||||
returned: success
|
||||
type: str
|
||||
labels:
|
||||
description:
|
||||
- The labels associated with this dataset. You can use these to organize and
|
||||
group your datasets .
|
||||
returned: success
|
||||
type: dict
|
||||
lastModifiedTime:
|
||||
description:
|
||||
- The time when this table was last modified, in milliseconds since the epoch.
|
||||
returned: success
|
||||
type: int
|
||||
location:
|
||||
description:
|
||||
- The geographic location where the table resides. This value is inherited from
|
||||
the dataset.
|
||||
returned: success
|
||||
type: str
|
||||
name:
|
||||
description:
|
||||
- Name of the table.
|
||||
returned: success
|
||||
type: str
|
||||
numBytes:
|
||||
description:
|
||||
- The size of this table in bytes, excluding any data in the streaming buffer.
|
||||
returned: success
|
||||
type: int
|
||||
numLongTermBytes:
|
||||
description:
|
||||
- The number of bytes in the table that are considered "long-term storage".
|
||||
returned: success
|
||||
type: int
|
||||
numRows:
|
||||
description:
|
||||
- The number of rows of data in this table, excluding any data in the streaming
|
||||
buffer.
|
||||
returned: success
|
||||
type: int
|
||||
type:
|
||||
description:
|
||||
- Describes the table type.
|
||||
returned: success
|
||||
type: str
|
||||
view:
|
||||
description:
|
||||
- The view definition.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
useLegacySql:
|
||||
description:
|
||||
- Specifies whether to use BigQuery's legacy SQL for this view .
|
||||
returned: success
|
||||
type: bool
|
||||
userDefinedFunctionResources:
|
||||
description:
|
||||
- Describes user-defined function resources used in the query.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
inlineCode:
|
||||
description:
|
||||
- An inline resource that contains code for a user-defined function
|
||||
(UDF). Providing a inline code resource is equivalent to providing
|
||||
a URI for a file containing the same code.
|
||||
returned: success
|
||||
type: str
|
||||
resourceUri:
|
||||
description:
|
||||
- A code resource to load from a Google Cloud Storage URI (gs://bucket/path).
|
||||
returned: success
|
||||
type: str
|
||||
timePartitioning:
|
||||
description:
|
||||
- If specified, configures time-based partitioning for this table.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
expirationMs:
|
||||
description:
|
||||
- Number of milliseconds for which to keep the storage for a partition.
|
||||
returned: success
|
||||
type: int
|
||||
type:
|
||||
description:
|
||||
- The only type supported is DAY, which will generate one partition per
|
||||
day.
|
||||
returned: success
|
||||
type: str
|
||||
streamingBuffer:
|
||||
description:
|
||||
- Contains information regarding this table's streaming buffer, if one is present.
|
||||
This field will be absent if the table is not being streamed to or if there
|
||||
is no data in the streaming buffer.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
estimatedBytes:
|
||||
description:
|
||||
- A lower-bound estimate of the number of bytes currently in the streaming
|
||||
buffer.
|
||||
returned: success
|
||||
type: int
|
||||
estimatedRows:
|
||||
description:
|
||||
- A lower-bound estimate of the number of rows currently in the streaming
|
||||
buffer.
|
||||
returned: success
|
||||
type: int
|
||||
oldestEntryTime:
|
||||
description:
|
||||
- Contains the timestamp of the oldest entry in the streaming buffer, in
|
||||
milliseconds since the epoch, if the streaming buffer is available.
|
||||
returned: success
|
||||
type: int
|
||||
schema:
|
||||
description:
|
||||
- Describes the schema of this table.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
fields:
|
||||
description:
|
||||
- Describes the fields in a table.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
description:
|
||||
description:
|
||||
- The field description. The maximum length is 1,024 characters.
|
||||
returned: success
|
||||
type: str
|
||||
fields:
|
||||
description:
|
||||
- Describes the nested schema fields if the type property is set to
|
||||
RECORD.
|
||||
returned: success
|
||||
type: list
|
||||
mode:
|
||||
description:
|
||||
- The field mode.
|
||||
returned: success
|
||||
type: str
|
||||
name:
|
||||
description:
|
||||
- The field name.
|
||||
returned: success
|
||||
type: str
|
||||
type:
|
||||
description:
|
||||
- The field data type.
|
||||
returned: success
|
||||
type: str
|
||||
encryptionConfiguration:
|
||||
description:
|
||||
- Custom encryption configuration.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
kmsKeyName:
|
||||
description:
|
||||
- Describes the Cloud KMS encryption key that will be used to protect destination
|
||||
BigQuery table. The BigQuery Service Account associated with your project
|
||||
requires access to this encryption key.
|
||||
returned: success
|
||||
type: str
|
||||
expirationTime:
|
||||
description:
|
||||
- The time when this table expires, in milliseconds since the epoch. If not
|
||||
present, the table will persist indefinitely.
|
||||
returned: success
|
||||
type: int
|
||||
externalDataConfiguration:
|
||||
description:
|
||||
- Describes the data format, location, and other properties of a table stored
|
||||
outside of BigQuery. By defining these properties, the data source can then
|
||||
be queried as if it were a standard BigQuery table.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
autodetect:
|
||||
description:
|
||||
- Try to detect schema and format options automatically. Any option specified
|
||||
explicitly will be honored.
|
||||
returned: success
|
||||
type: bool
|
||||
compression:
|
||||
description:
|
||||
- The compression type of the data source.
|
||||
returned: success
|
||||
type: str
|
||||
ignoreUnknownValues:
|
||||
description:
|
||||
- Indicates if BigQuery should allow extra values that are not represented
|
||||
in the table schema .
|
||||
returned: success
|
||||
type: bool
|
||||
maxBadRecords:
|
||||
description:
|
||||
- The maximum number of bad records that BigQuery can ignore when reading
|
||||
data .
|
||||
returned: success
|
||||
type: int
|
||||
sourceFormat:
|
||||
description:
|
||||
- The data format.
|
||||
returned: success
|
||||
type: str
|
||||
sourceUris:
|
||||
description:
|
||||
- 'The fully-qualified URIs that point to your data in Google Cloud. For
|
||||
Google Cloud Storage URIs: Each URI can contain one ''*'' wildcard character
|
||||
and it must come after the ''bucket'' name. Size limits related to load
|
||||
jobs apply to external data sources. For Google Cloud Bigtable URIs: Exactly
|
||||
one URI can be specified and it has be a fully specified and valid HTTPS
|
||||
URL for a Google Cloud Bigtable table. For Google Cloud Datastore backups,
|
||||
exactly one URI can be specified. Also, the ''*'' wildcard character is
|
||||
not allowed.'
|
||||
returned: success
|
||||
type: list
|
||||
schema:
|
||||
description:
|
||||
- The schema for the data. Schema is required for CSV and JSON formats.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
fields:
|
||||
description:
|
||||
- Describes the fields in a table.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
description:
|
||||
description:
|
||||
- The field description.
|
||||
returned: success
|
||||
type: str
|
||||
fields:
|
||||
description:
|
||||
- Describes the nested schema fields if the type property is set
|
||||
to RECORD .
|
||||
returned: success
|
||||
type: list
|
||||
mode:
|
||||
description:
|
||||
- Field mode.
|
||||
returned: success
|
||||
type: str
|
||||
name:
|
||||
description:
|
||||
- Field name.
|
||||
returned: success
|
||||
type: str
|
||||
type:
|
||||
description:
|
||||
- Field data type.
|
||||
returned: success
|
||||
type: str
|
||||
googleSheetsOptions:
|
||||
description:
|
||||
- Additional options if sourceFormat is set to GOOGLE_SHEETS.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
skipLeadingRows:
|
||||
description:
|
||||
- The number of rows at the top of a Google Sheet that BigQuery will
|
||||
skip when reading the data.
|
||||
returned: success
|
||||
type: int
|
||||
csvOptions:
|
||||
description:
|
||||
- Additional properties to set if sourceFormat is set to CSV.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
allowJaggedRows:
|
||||
description:
|
||||
- Indicates if BigQuery should accept rows that are missing trailing
|
||||
optional columns .
|
||||
returned: success
|
||||
type: bool
|
||||
allowQuotedNewlines:
|
||||
description:
|
||||
- Indicates if BigQuery should allow quoted data sections that contain
|
||||
newline characters in a CSV file .
|
||||
returned: success
|
||||
type: bool
|
||||
encoding:
|
||||
description:
|
||||
- The character encoding of the data.
|
||||
returned: success
|
||||
type: str
|
||||
fieldDelimiter:
|
||||
description:
|
||||
- The separator for fields in a CSV file.
|
||||
returned: success
|
||||
type: str
|
||||
quote:
|
||||
description:
|
||||
- The value that is used to quote data sections in a CSV file.
|
||||
returned: success
|
||||
type: str
|
||||
skipLeadingRows:
|
||||
description:
|
||||
- The number of rows at the top of a CSV file that BigQuery will skip
|
||||
when reading the data.
|
||||
returned: success
|
||||
type: int
|
||||
bigtableOptions:
|
||||
description:
|
||||
- Additional options if sourceFormat is set to BIGTABLE.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
ignoreUnspecifiedColumnFamilies:
|
||||
description:
|
||||
- If field is true, then the column families that are not specified
|
||||
in columnFamilies list are not exposed in the table schema .
|
||||
returned: success
|
||||
type: bool
|
||||
readRowkeyAsString:
|
||||
description:
|
||||
- If field is true, then the rowkey column families will be read and
|
||||
converted to string.
|
||||
returned: success
|
||||
type: bool
|
||||
columnFamilies:
|
||||
description:
|
||||
- List of column families to expose in the table schema along with their
|
||||
types.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
columns:
|
||||
description:
|
||||
- Lists of columns that should be exposed as individual fields as
|
||||
opposed to a list of (column name, value) pairs.
|
||||
returned: success
|
||||
type: complex
|
||||
contains:
|
||||
encoding:
|
||||
description:
|
||||
- The encoding of the values when the type is not STRING.
|
||||
returned: success
|
||||
type: str
|
||||
fieldName:
|
||||
description:
|
||||
- If the qualifier is not a valid BigQuery field identifier,
|
||||
a valid identifier must be provided as the column field name
|
||||
and is used as field name in queries.
|
||||
returned: success
|
||||
type: str
|
||||
onlyReadLatest:
|
||||
description:
|
||||
- If this is set, only the latest version of value in this column
|
||||
are exposed .
|
||||
returned: success
|
||||
type: bool
|
||||
qualifierString:
|
||||
description:
|
||||
- Qualifier of the column.
|
||||
returned: success
|
||||
type: str
|
||||
type:
|
||||
description:
|
||||
- The type to convert the value in cells of this column.
|
||||
returned: success
|
||||
type: str
|
||||
encoding:
|
||||
description:
|
||||
- The encoding of the values when the type is not STRING.
|
||||
returned: success
|
||||
type: str
|
||||
familyId:
|
||||
description:
|
||||
- Identifier of the column family.
|
||||
returned: success
|
||||
type: str
|
||||
onlyReadLatest:
|
||||
description:
|
||||
- If this is set only the latest version of value are exposed for
|
||||
all columns in this column family .
|
||||
returned: success
|
||||
type: bool
|
||||
type:
|
||||
description:
|
||||
- The type to convert the value in cells of this column family.
|
||||
returned: success
|
||||
type: str
|
||||
dataset:
|
||||
description:
|
||||
- Name of the dataset.
|
||||
returned: success
|
||||
type: str
|
||||
'''
|
||||
|
||||
################################################################################
|
||||
# Imports
|
||||
################################################################################
|
||||
from ansible.module_utils.gcp_utils import navigate_hash, GcpSession, GcpModule, GcpRequest
|
||||
import json
|
||||
|
||||
################################################################################
|
||||
# Main
|
||||
################################################################################
|
||||
|
||||
|
||||
def main():
|
||||
module = GcpModule(
|
||||
argument_spec=dict(
|
||||
dataset=dict(type='str')
|
||||
)
|
||||
)
|
||||
|
||||
if not module.params['scopes']:
|
||||
module.params['scopes'] = ['https://www.googleapis.com/auth/bigquery']
|
||||
|
||||
items = fetch_list(module, collection(module))
|
||||
if items.get('tables'):
|
||||
items = items.get('tables')
|
||||
else:
|
||||
items = []
|
||||
return_value = {
|
||||
'items': items
|
||||
}
|
||||
module.exit_json(**return_value)
|
||||
|
||||
|
||||
def collection(module):
|
||||
return "https://www.googleapis.com/bigquery/v2/projects/{project}/datasets/{dataset}/tables".format(**module.params)
|
||||
|
||||
|
||||
def fetch_list(module, link):
|
||||
auth = GcpSession(module, 'bigquery')
|
||||
response = auth.get(link)
|
||||
return return_if_object(module, response)
|
||||
|
||||
|
||||
def return_if_object(module, response):
|
||||
# If not found, return nothing.
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
|
||||
# If no content, return nothing.
|
||||
if response.status_code == 204:
|
||||
return None
|
||||
|
||||
try:
|
||||
module.raise_for_status(response)
|
||||
result = response.json()
|
||||
except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst:
|
||||
module.fail_json(msg="Invalid JSON response with error: %s" % inst)
|
||||
|
||||
if navigate_hash(result, ['error', 'errors']):
|
||||
module.fail_json(msg=navigate_hash(result, ['error', 'errors']))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue