From 9730157525c8cd539c2f657f40c34b73bb189e44 Mon Sep 17 00:00:00 2001 From: James Cammarata Date: Mon, 10 Mar 2014 16:06:52 -0500 Subject: [PATCH] Validate SSL certs accessed through urllib* * Adds another module utility file which generalizes the access of urls via the urllib* libraries. * Adds a new spec generator for common arguments. * Makes the user-agent string configurable. Fixes #6211 --- examples/ansible.cfg | 14 ++ lib/ansible/constants.py | 4 + lib/ansible/module_utils/basic.py | 10 +- lib/ansible/module_utils/ec2.py | 28 +++ lib/ansible/module_utils/known_hosts.py | 28 +++ lib/ansible/module_utils/rax.py | 29 ++- lib/ansible/module_utils/urls.py | 262 ++++++++++++++++++++++++ library/cloud/ec2_facts | 24 +-- library/database/riak | 21 +- library/monitoring/airbrake_deployment | 41 ++-- library/monitoring/boundary_meter | 61 ++---- library/monitoring/datadog_event | 18 +- library/monitoring/newrelic_deployment | 48 +---- library/monitoring/pagerduty | 35 ++-- library/net_infrastructure/dnsmadeeasy | 32 +-- library/net_infrastructure/netscaler | 40 ++-- library/network/get_url | 116 ++--------- library/notification/flowdock | 31 +-- library/notification/grove | 7 +- library/notification/hipchat | 35 +--- library/packaging/apt_key | 20 +- library/packaging/rpm_key | 17 +- library/source_control/github_hooks | 79 +++---- 23 files changed, 598 insertions(+), 402 deletions(-) create mode 100644 lib/ansible/module_utils/urls.py diff --git a/examples/ansible.cfg b/examples/ansible.cfg index 5b23e101269..fde76e5b558 100644 --- a/examples/ansible.cfg +++ b/examples/ansible.cfg @@ -98,6 +98,20 @@ filter_plugins = /usr/share/ansible_plugins/filter_plugins # set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 #nocolor = 1 +# the CA certificate path used for validating SSL certs. This path +# should exist on the controlling node, not the target nodes +# common locations: +# RHEL/CentOS: /etc/pki/tls/certs/ca-bundle.crt +# Fedora : /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +# Ubuntu : /usr/share/ca-certificates/cacert.org/cacert.org.crt +#ca_file_path = + +# the http user-agent string to use when fetching urls. Some web server +# operators block the default urllib user agent as it is frequently used +# by malicious attacks/scripts, so we set it to something unique to +# avoid issues. +#http_user_agent = ansible-agent + [paramiko_connection] # uncomment this line to cause the paramiko connection plugin to not record new host diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 6bf87e51f8a..9d3f37c180d 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -143,6 +143,10 @@ DEFAULT_VARS_PLUGIN_PATH = get_config(p, DEFAULTS, 'vars_plugins', ' DEFAULT_FILTER_PLUGIN_PATH = get_config(p, DEFAULTS, 'filter_plugins', 'ANSIBLE_FILTER_PLUGINS', '/usr/share/ansible_plugins/filter_plugins') DEFAULT_LOG_PATH = shell_expand_path(get_config(p, DEFAULTS, 'log_path', 'ANSIBLE_LOG_PATH', '')) +# URL Arguments for generic module urllib2 use +DEFAULT_HTTP_USER_AGENT = get_config(p, DEFAULTS, 'http_user_agent', 'ANSIBLE_HTTP_USER_AGENT', 'ansible-agent') +DEFAULT_CA_FILE_PATH = shell_expand_path(get_config(p, DEFAULTS, 'ca_file_path', 'ANSIBLE_CA_FILE_PATH', '')) + ANSIBLE_NOCOLOR = get_config(p, DEFAULTS, 'nocolor', 'ANSIBLE_NOCOLOR', None, boolean=True) ANSIBLE_NOCOWS = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None, boolean=True) DISPLAY_SKIPPED_HOSTS = get_config(p, DEFAULTS, 'display_skipped_hosts', 'DISPLAY_SKIPPED_HOSTS', True, boolean=True) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 540efeb4bfc..8025563e58e 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -60,6 +60,7 @@ import grp import pwd import platform import errno +import tempfile try: import json @@ -115,6 +116,7 @@ FILE_COMMON_ARGUMENTS=dict( remote_src = dict(), # used by assemble ) + def get_platform(): ''' what's the platform? example: Linux is a platform. ''' return platform.system() @@ -189,7 +191,7 @@ class AnsibleModule(object): os.environ['LANG'] = MODULE_LANG (self.params, self.args) = self._load_params() - self._legal_inputs = [ 'CHECKMODE', 'NO_LOG' ] + self._legal_inputs = ['CHECKMODE', 'NO_LOG'] self.aliases = self._handle_aliases() @@ -572,8 +574,9 @@ class AnsibleModule(object): def _check_invalid_arguments(self): for (k,v) in self.params.iteritems(): - if k in ('CHECKMODE', 'NO_LOG'): - continue + # these should be in legal inputs already + #if k in ('CHECKMODE', 'NO_LOG'): + # continue if k not in self._legal_inputs: self.fail_json(msg="unsupported parameter for module: %s" % k) @@ -1093,4 +1096,3 @@ class AnsibleModule(object): break return '%.2f %s' % (float(size)/ limit, suffix) - diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index 9156df766b2..58291c2d5d5 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -1,3 +1,31 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan , 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + try: from distutils.version import LooseVersion HAS_LOOSE_VERSION = True diff --git a/lib/ansible/module_utils/known_hosts.py b/lib/ansible/module_utils/known_hosts.py index 000db9d1e62..36f5b87fff5 100644 --- a/lib/ansible/module_utils/known_hosts.py +++ b/lib/ansible/module_utils/known_hosts.py @@ -1,3 +1,31 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan , 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + def add_git_host_key(module, url, accept_hostkey=True): """ idempotently add a git url hostkey """ diff --git a/lib/ansible/module_utils/rax.py b/lib/ansible/module_utils/rax.py index 84e5686d24f..98623c7d38e 100644 --- a/lib/ansible/module_utils/rax.py +++ b/lib/ansible/module_utils/rax.py @@ -1,5 +1,32 @@ -import os +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan , 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os def rax_argument_spec(): return dict( diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py new file mode 100644 index 00000000000..f251c6b407f --- /dev/null +++ b/lib/ansible/module_utils/urls.py @@ -0,0 +1,262 @@ +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Copyright (c), Michael DeHaan , 2012-2013 +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +try: + import urllib + HAS_URLLIB = True +except: + HAS_URLLIB = False + +try: + import urllib2 + HAS_URLLIB2 = True +except: + HAS_URLLIB2 = False + +try: + import urlparse + HAS_URLPARSE = True +except: + HAS_URLPARSE = False + +try: + import ssl + HAS_SSL=True +except: + HAS_SSL=False + + +class RequestWithMethod(urllib2.Request): + ''' + Workaround for using DELETE/PUT/etc with urllib2 + Originally contained in library/net_infrastructure/dnsmadeeasy + ''' + + def __init__(self, url, method, data=None, headers={}): + self._method = method + urllib2.Request.__init__(self, url, data, headers) + + def get_method(self): + if self._method: + return self._method + else: + return urllib2.Request.get_method(self) + + +class SSLValidationHandler(urllib2.BaseHandler): + ''' + A custom handler class for SSL validation. + + Based on: + http://stackoverflow.com/questions/1087227/validate-ssl-certificates-with-python + http://techknack.net/python-urllib2-handlers/ + ''' + + def __init__(self, module, hostname, port, ca_cert=None): + self.module = module + self.hostname = hostname + self.port = port + self.ca_cert = ca_cert + + def get_ca_cert(self): + # tries to find a valid CA cert in one of the + # standard locations for the current distribution + + if self.ca_cert and os.path.exists(self.ca_cert): + # the user provided a custom CA cert (ie. one they + # uploaded themselves), so use it + return self.ca_cert + + ca_cert = None + platform = get_platform() + distribution = get_distribution() + if platform == 'Linux': + if distribution in ('Fedora',): + ca_cert = '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem' + elif distribution in ('RHEL','CentOS','ScientificLinux'): + ca_cert = '/etc/pki/tls/certs/ca-bundle.crt' + elif distribution in ('Ubuntu','Debian'): + ca_cert = '/usr/share/ca-certificates/cacert.org/cacert.org.crt' + elif platform == 'FreeBSD': + ca_cert = '/usr/local/share/certs/ca-root.crt' + elif platform == 'OpenBSD': + ca_cert = '/etc/ssl/cert.pem' + elif platform == 'NetBSD': + ca_cert = '/etc/openssl/certs/ca-cert.pem' + elif platform == 'SunOS': + # FIXME? + pass + elif platform == 'AIX': + # FIXME? + pass + + if ca_cert and os.path.exists(ca_cert): + return ca_cert + elif os.path.exists('/etc/ansible/ca-cert.pem'): + # fall back to a user-deployed cert in a standard + # location if the OS platform one is not available + return '/etc/ansible/ca-cert.pem' + else: + # CA cert isn't available, no validation + return None + + def http_request(self, req): + try: + server_cert = ssl.get_server_certificate((self.hostname, self.port), ca_certs=self.get_ca_cert()) + except ssl.SSLError: + self.module.fail_json(msg='failed to validate the SSL certificate for %s:%s. You can use validate_certs=no, however this is unsafe and not recommended' % (self.hostname, self.port)) + return req + + https_request = http_request + + +def url_argument_spec(): + ''' + Creates an argument spec that can be used with any module + that will be requesting content via urllib/urllib2 + ''' + return dict( + url = dict(), + force = dict(default='no', aliases=['thirsty'], type='bool'), + http_agent = dict(default='ansible-httpget'), + use_proxy = dict(default='yes', type='bool'), + validate_certs = dict(default='yes', type='bool'), + ) + + +def fetch_url(module, url, data=None, headers=None, method=None, + use_proxy=False, validate_certs=True, force=False, last_mod_time=None, timeout=10): + ''' + Fetches a file from an HTTP/FTP server using urllib2 + ''' + + if not HAS_URLLIB: + module.fail_json(msg='urllib is not installed') + if not HAS_URLLIB2: + module.fail_json(msg='urllib2 is not installed') + elif not HAS_URLPARSE: + module.fail_json(msg='urlparse is not installed') + + r = None + handlers = [] + info = dict(url=url) + + parsed = urlparse.urlparse(url) + if parsed[0] == 'https': + if not HAS_SSL and validate_certs: + module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended') + elif validate_certs: + # do the cert validation + netloc = parsed[1] + if '@' in netloc: + netloc = netloc.split('@', 1)[1] + if ':' in netloc: + hostname, port = netloc.split(':', 1) + else: + hostname = netloc + port = 443 + # create the SSL validation handler and + # add it to the list of handlers + ssl_handler = SSLValidationHandler(module, hostname, port) + handlers.append(ssl_handler) + + if '@' in parsed[1]: + credentials, netloc = parsed[1].split('@', 1) + if ':' in credentials: + username, password = credentials.split(':', 1) + else: + username = credentials + password = '' + parsed = list(parsed) + parsed[1] = netloc + + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + # this creates a password manager + passman.add_password(None, netloc, username, password) + # because we have put None at the start it will always + # use this username/password combination for urls + # for which `theurl` is a super-url + + authhandler = urllib2.HTTPBasicAuthHandler(passman) + # create the AuthHandler + handlers.append(authhandler) + + #reconstruct url without credentials + url = urlparse.urlunparse(parsed) + + if not use_proxy: + proxyhandler = urllib2.ProxyHandler({}) + handlers.append(proxyhandler) + + opener = urllib2.build_opener(*handlers) + urllib2.install_opener(opener) + + if method: + if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'): + module.fail_json(msg='invalid HTTP request method; %s' % method.upper()) + request = RequestWithMethod(url, method.upper(), data) + else: + request = urllib2.Request(url, data) + + # add the custom agent header, to help prevent issues + # with sites that block the default urllib agent string + request.add_header('User-agent', module.params.get('http_agent')) + + # if we're ok with getting a 304, set the timestamp in the + # header, otherwise make sure we don't get a cached copy + if last_mod_time and not force: + tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') + request.add_header('If-Modified-Since', tstamp) + else: + request.add_header('cache-control', 'no-cache') + + # user defined headers now, which may override things we've set above + if headers: + if not isinstance(headers, dict): + module.fail_json("headers provided to fetch_url() must be a dict") + for header in headers: + request.add_header(header, headers[header]) + + try: + if sys.version_info < (2,6,0): + # urlopen in python prior to 2.6.0 did not + # have a timeout parameter + r = urllib2.urlopen(request, None) + else: + r = urllib2.urlopen(request, None, timeout) + info.update(r.info()) + info['url'] = r.geturl() # The URL goes in too, because of redirects. + info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200)) + except urllib2.HTTPError, e: + info.update(dict(msg=str(e), status=e.code)) + except urllib2.URLError, e: + code = int(getattr(e, 'code', -1)) + info.update(dict(msg="Request failed: %s" % str(e), status=code)) + + return r, info + diff --git a/library/cloud/ec2_facts b/library/cloud/ec2_facts index 1c17fa5b717..c6a6670a58b 100644 --- a/library/cloud/ec2_facts +++ b/library/cloud/ec2_facts @@ -41,7 +41,6 @@ EXAMPLES = ''' when: ansible_ec2_instance_type == "t1.micro" ''' -import urllib2 import socket import re @@ -62,7 +61,8 @@ class Ec2Metadata(object): 'us-west-1', 'us-west-2') - def __init__(self, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None): + def __init__(self, module, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None): + self.module = module self.uri_meta = ec2_metadata_uri or self.ec2_metadata_uri self.uri_user = ec2_userdata_uri or self.ec2_userdata_uri self.uri_ssh = ec2_sshdata_uri or self.ec2_sshdata_uri @@ -70,12 +70,9 @@ class Ec2Metadata(object): self._prefix = 'ansible_ec2_%s' def _fetch(self, url): - try: - return urllib2.urlopen(url).read() - except urllib2.HTTPError: - return - except urllib2.URLError: - return + self.module.fail_json(msg="url is %s" % url) + (response, info) = fetch_url(self.module, url, force=True) + return response.read() def _mangle_fields(self, fields, uri, filter_patterns=['public-keys-0']): new_fields = {} @@ -150,17 +147,20 @@ class Ec2Metadata(object): return data def main(): - - ec2_facts = Ec2Metadata().run() - ec2_facts_result = dict(changed=False, ansible_facts=ec2_facts) + argument_spec = url_argument_spec() module = AnsibleModule( - argument_spec = dict(), + argument_spec = argument_spec, supports_check_mode = True, ) + + ec2_facts = Ec2Metadata(module).run() + ec2_facts_result = dict(changed=False, ansible_facts=ec2_facts) + module.exit_json(**ec2_facts_result) # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * main() diff --git a/library/database/riak b/library/database/riak index 53faba6e983..e0a7552f0ae 100644 --- a/library/database/riak +++ b/library/database/riak @@ -138,24 +138,13 @@ def main(): while True: if time.time() > timeout: module.fail_json(msg='Timeout, could not fetch Riak stats.') - try: - if sys.version_info<(2,6,0): - stats_raw = urllib2.urlopen( - 'http://%s/stats' % (http_conn), None).read() - else: - stats_raw = urllib2.urlopen( - 'http://%s/stats' % (http_conn), None, 5).read() + (response, info) = fetch_url(module, 'http://%s/stats' % (http_conn), force=True, timeout=5) + if info['status'] == 200: + stats_raw = response.read() break - except urllib2.HTTPError, e: - time.sleep(5) - except urllib2.URLError, e: - time.sleep(5) - except socket.timeout: - time.sleep(5) - except Exception, e: - module.fail_json(msg='Could not fetch Riak stats: %s' % e) + time.sleep(5) -# here we attempt to load those stats, + # here we attempt to load those stats, try: stats = json.loads(stats_raw) except: diff --git a/library/monitoring/airbrake_deployment b/library/monitoring/airbrake_deployment index 8a4a834be7c..6a83459906a 100644 --- a/library/monitoring/airbrake_deployment +++ b/library/monitoring/airbrake_deployment @@ -52,6 +52,13 @@ options: - Optional URL to submit the notification to. Use to send notifications to Airbrake-compliant tools like Errbit. required: false default: https://airbrake.io/deploys + validate_certs: + description: + - If C(no), SSL certificates for the target url will not be validated. This should only be used + on personally controlled sites using self-signed certificates. + required: false + default: 'yes' + choices: ['yes', 'no'] # informational: requirements for nodes requirements: [ urllib, urllib2 ] @@ -64,29 +71,12 @@ EXAMPLES = ''' revision=4.2 ''' -HAS_URLLIB = True -try: - import urllib -except ImportError: - HAS_URLLIB = False - -HAS_URLLIB2 = True -try: - import urllib2 -except ImportError: - HAS_URLLIB2 = False - # =========================================== # Module execution. # def main(): - if not HAS_URLLIB: - module.fail_json(msg="urllib is not installed") - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - module = AnsibleModule( argument_spec=dict( token=dict(required=True), @@ -95,6 +85,7 @@ def main(): repo=dict(required=False), revision=dict(required=False), url=dict(required=False, default='https://api.airbrake.io/deploys.txt') + validate_certs=dict(default='yes', type='bool'), ), supports_check_mode=True ) @@ -123,18 +114,16 @@ def main(): module.exit_json(changed=True) # Send the data to airbrake - try: - req = urllib2.Request(url, urllib.urlencode(params)) - result=urllib2.urlopen(req) - except Exception, e: - module.fail_json(msg="unable to update airbrake via %s?%s : %s" % (url, urllib.urlencode(params), e)) + data = urllib.urlencode(params) + response, info = fetch_url(module, url, data=data, validate_certs=module.params['validate_certs']) + if info['status'] == 200: + module.exit_json(changed=True) else: - if result.code == 200: - module.exit_json(changed=True) - else: - module.fail_json(msg="HTTP result code: %d connecting to %s" % (result.code, url)) + module.fail_json(msg="HTTP result code: %d connecting to %s" % (info['status'], url)) # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/monitoring/boundary_meter b/library/monitoring/boundary_meter index 202dfd03ae3..3c9f90a4ce9 100644 --- a/library/monitoring/boundary_meter +++ b/library/monitoring/boundary_meter @@ -24,7 +24,6 @@ along with Ansible. If not, see . import json import datetime -import urllib2 import base64 import os @@ -74,12 +73,6 @@ EXAMPLES=''' ''' -try: - import urllib2 - HAS_URLLIB2 = True -except ImportError: - HAS_URLLIB2 = False - api_host = "api.boundary.com" config_directory = "/etc/bprobe" @@ -101,7 +94,7 @@ def build_url(name, apiid, action, meter_id=None, cert_type=None): elif action == "delete": return "https://%s/%s/meters/%s" % (api_host, apiid, meter_id) -def http_request(name, apiid, apikey, action, meter_id=None, cert_type=None): +def http_request(module, name, apiid, apikey, action, data=None, meter_id=None, cert_type=None): if meter_id is None: url = build_url(name, apiid, action) @@ -111,11 +104,11 @@ def http_request(name, apiid, apikey, action, meter_id=None, cert_type=None): else: url = build_url(name, apiid, action, meter_id, cert_type) - auth = auth_encode(apikey) - request = urllib2.Request(url) - request.add_header("Authorization", "Basic %s" % (auth)) - request.add_header("Content-Type", "application/json") - return request + headers = dict() + headers["Authorization"] = "Basic %s" % auth_encode(apikey) + headers["Content-Type"] = "application/json" + + return fetch_url(module, url, data=data, headers=headers) def create_meter(module, name, apiid, apikey): @@ -126,14 +119,10 @@ def create_meter(module, name, apiid, apikey): module.exit_json(status="Meter " + name + " already exists",changed=False) else: # If it doesn't exist, create it - request = http_request(name, apiid, apikey, action="create") - # A create request seems to need a json body with the name of the meter in it body = '{"name":"' + name + '"}' - request.add_data(body) + response, info = http_request(module, name, apiid, apikey, data=body, action="create") - try: - result = urllib2.urlopen(request) - except urllib2.URLError, e: + if info['status'] != 200: module.fail_json(msg="Failed to connect to api host to create meter") # If the config directory doesn't exist, create it @@ -160,15 +149,13 @@ def create_meter(module, name, apiid, apikey): def search_meter(module, name, apiid, apikey): - request = http_request(name, apiid, apikey, action="search") + response, info = http_request(module, name, apiid, apikey, action="search") - try: - result = urllib2.urlopen(request) - except urllib2.URLError, e: + if info['status'] != 200: module.fail_json("Failed to connect to api host to search for meter") # Return meters - return json.loads(result.read()) + return json.loads(response.read()) def get_meter_id(module, name, apiid, apikey): # In order to delete the meter we need its id @@ -186,16 +173,9 @@ def delete_meter(module, name, apiid, apikey): if meter_id is None: return 1, "Meter does not exist, so can't delete it" else: - action = "delete" - request = http_request(name, apiid, apikey, action, meter_id) - # See http://stackoverflow.com/questions/4511598/how-to-make-http-delete-method-using-urllib2 - # urllib2 only does GET or POST I believe, but here we need delete - request.get_method = lambda: 'DELETE' - - try: - result = urllib2.urlopen(request) - except urllib2.URLError, e: - module.fail_json("Failed to connect to api host to delete meter") + response, info = http_request(module, name, apiid, apikey, action, meter_id) + if info['status'] != 200: + module.fail_json("Failed to delete meter") # Each new meter gets a new key.pem and ca.pem file, so they should be deleted types = ['cert', 'key'] @@ -214,17 +194,14 @@ def download_request(module, name, apiid, apikey, cert_type): if meter_id is not None: action = "certificates" - request = http_request(name, apiid, apikey, action, meter_id, cert_type) - - try: - result = urllib2.urlopen(request) - except urllib2.URLError, e: + response, info = http_request(module, name, apiid, apikey, action, meter_id, cert_type) + if info['status'] != 200: module.fail_json("Failed to connect to api host to download certificate") if result: try: cert_file_path = '%s/%s.pem' % (config_directory,cert_type) - body = result.read() + body = response.read() cert_file = open(cert_file_path, 'w') cert_file.write(body) cert_file.close @@ -238,9 +215,6 @@ def download_request(module, name, apiid, apikey, cert_type): def main(): - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - module = AnsibleModule( argument_spec=dict( state=dict(required=True, choices=['present', 'absent']), @@ -268,5 +242,6 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * main() diff --git a/library/monitoring/datadog_event b/library/monitoring/datadog_event index 629e86e98ab..878aee6d343 100644 --- a/library/monitoring/datadog_event +++ b/library/monitoring/datadog_event @@ -67,7 +67,6 @@ datadog_event: title="Testing from ansible" text="Test!" ''' import socket -from urllib2 import urlopen, Request, URLError def main(): module = AnsibleModule( @@ -97,8 +96,7 @@ def main(): post_event(module) def post_event(module): - uri = "https://app.datadoghq.com/api/v1/events?api_key=" + \ - module.params['api_key'] + uri = "https://app.datadoghq.com/api/v1/events?api_key=%s" % module.params['api_key'] body = dict( title=module.params['title'], @@ -117,22 +115,20 @@ def post_event(module): json_body = module.jsonify(body) headers = {"Content-Type": "application/json"} - request = Request(uri, json_body, headers, unverifiable=True) - try: - response = urlopen(request) + (response, info) = fetch_url(module, uri, data=json_body, headers=headers) + if info['status'] == 200: response_body = response.read() response_json = module.from_json(response_body) if response_json['status'] == 'ok': module.exit_json(changed=True) else: module.fail_json(msg=response) - - except URLError, e: - module.fail_json(msg="URL error: %s." % e) - except socket.error, e: - module.fail_json(msg="Socket error: %s to %s" % (e, uri)) + else: + module.fail_json(**info) # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/monitoring/newrelic_deployment b/library/monitoring/newrelic_deployment index de64651969c..08132722e1d 100644 --- a/library/monitoring/newrelic_deployment +++ b/library/monitoring/newrelic_deployment @@ -75,29 +75,12 @@ EXAMPLES = ''' revision=1.0 ''' -HAS_URLLIB = True -try: - import urllib -except ImportError: - HAS_URLLIB = False - -HAS_URLLIB2 = True -try: - import urllib2 -except ImportError: - HAS_URLLIB2 = False - # =========================================== # Module execution. # def main(): - if not HAS_URLLIB: - module.fail_json(msg="urllib is not installed") - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - module = AnsibleModule( argument_spec=dict( token=dict(required=True), @@ -134,29 +117,20 @@ def main(): module.exit_json(changed=True) # Send the data to NewRelic - try: - req = urllib2.Request("https://rpm.newrelic.com/deployments.xml", urllib.urlencode(params)) - req.add_header('x-api-key',module.params["token"]) - result=urllib2.urlopen(req) - # urlopen behaves differently in python 2.4 and 2.6 so we handle - # both cases here. In python 2.4 it throws an exception if the - # return code is anything other than a 200. In python 2.6 it - # doesn't throw an exception for any 2xx return codes. In both - # cases we expect newrelic should return a 201 on success. So - # to handle both cases, both the except & else cases below are - # effectively identical. - except Exception, e: - if e.code == 201: - module.exit_json(changed=True) - else: - module.fail_json(msg="unable to update newrelic: %s" % e) + url = "https://rpm.newrelic.com/deployments.xml" + data = urllib.urlencode(params) + headers = { + 'x-api-key': module.params["token"], + } + response, info = fetch_url(module, url, data=data, headers=headers) + if info['status'] in (200, 201): + module.exit_json(changed=True) else: - if result.code == 201: - module.exit_json(changed=True) - else: - module.fail_json(msg="result code: %d" % result.code) + module.fail_json(msg="unable to update newrelic: %s" % info['msg']) # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/monitoring/pagerduty b/library/monitoring/pagerduty index bfd0573f4de..9a7f21d0779 100644 --- a/library/monitoring/pagerduty +++ b/library/monitoring/pagerduty @@ -87,24 +87,23 @@ EXAMPLES=''' import json import datetime -import urllib2 import base64 -def ongoing(name, user, passwd): +def ongoing(module, name, user, passwd): url = "https://" + name + ".pagerduty.com/api/v1/maintenance_windows/ongoing" auth = base64.encodestring('%s:%s' % (user, passwd)).replace('\n', '') + headers = {"Authorization": "Basic %s" % auth} - req = urllib2.Request(url) - req.add_header("Authorization", "Basic %s" % auth) - res = urllib2.urlopen(req) - out = res.read() + response, info = fetch_url(module, url, headers=headers) + if info['status'] != 200: + module.fail_json(msg="failed to lookup the ongoing window: %s" % info['msg']) - return False, out + return False, response.read() -def create(name, user, passwd, service, hours, desc): +def create(module, name, user, passwd, service, hours, desc): now = datetime.datetime.utcnow() later = now + datetime.timedelta(hours=int(hours)) @@ -113,15 +112,17 @@ def create(name, user, passwd, service, hours, desc): url = "https://" + name + ".pagerduty.com/api/v1/maintenance_windows" auth = base64.encodestring('%s:%s' % (user, passwd)).replace('\n', '') + headers = { + 'Authorization': 'Basic %s' % auth, + 'Content-Type' : 'application/json', + } data = json.dumps({'maintenance_window': {'start_time': start, 'end_time': end, 'description': desc, 'service_ids': [service]}}) - req = urllib2.Request(url, data) - req.add_header("Authorization", "Basic %s" % auth) - req.add_header('Content-Type', 'application/json') - res = urllib2.urlopen(req) - out = res.read() + response, info = fetch_url(module, url, data=data, headers=headers, method='POST') + if info['status'] != 200: + module.fail_json(msg="failed to create the window: %s" % info['msg']) - return False, out + return False, response.read() def main(): @@ -149,10 +150,10 @@ def main(): if state == "running" or state == "started": if not service: module.fail_json(msg="service not specified") - (rc, out) = create(name, user, passwd, service, hours, desc) + (rc, out) = create(module, name, user, passwd, service, hours, desc) if state == "ongoing": - (rc, out) = ongoing(name, user, passwd) + (rc, out) = ongoing(module, name, user, passwd) if rc != 0: module.fail_json(msg="failed", result=out) @@ -161,4 +162,6 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/net_infrastructure/dnsmadeeasy b/library/net_infrastructure/dnsmadeeasy index d4af13e884a..9e2c14480eb 100644 --- a/library/net_infrastructure/dnsmadeeasy +++ b/library/net_infrastructure/dnsmadeeasy @@ -106,8 +106,6 @@ EXAMPLES = ''' IMPORT_ERROR = None try: - import urllib - import urllib2 import json from time import strftime, gmtime import hashlib @@ -115,22 +113,6 @@ try: except ImportError, e: IMPORT_ERROR = str(e) - -class RequestWithMethod(urllib2.Request): - - """Workaround for using DELETE/PUT/etc with urllib2""" - - def __init__(self, url, method, data=None, headers={}): - self._method = method - urllib2.Request.__init__(self, url, data, headers) - - def get_method(self): - if self._method: - return self._method - else: - return urllib2.Request.get_method(self) - - class DME2: def __init__(self, apikey, secret, domain, module): @@ -169,16 +151,10 @@ class DME2: url = self.baseurl + resource if data and not isinstance(data, basestring): data = urllib.urlencode(data) - request = RequestWithMethod(url, method, data, self._headers()) - try: - response = urllib2.urlopen(request) - except urllib2.HTTPError, e: - self.module.fail_json( - msg="%s returned %s, with body: %s" % (url, e.code, e.read())) - except Exception, e: - self.module.fail_json( - msg="Failed contacting: %s : Exception %s" % (url, e.message())) + response, info = fetch_url(self.module, url, data=data, method=method) + if info['status'] not in (200, 201, 204): + self.module.fail_json(msg="%s returned %s, with body: %s" % (url, info['status'], info['msg'])) try: return json.load(response) @@ -338,4 +314,6 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/net_infrastructure/netscaler b/library/net_infrastructure/netscaler index 1aa370895d5..4756d90abdc 100644 --- a/library/net_infrastructure/netscaler +++ b/library/net_infrastructure/netscaler @@ -73,6 +73,14 @@ options: default: server choices: ["server", "service"] aliases: [] + validate_certs: + description: + - If C(no), SSL certificates for the target url will not be validated. This should only be used + on personally controlled sites using self-signed certificates. + required: false + default: 'yes' + choices: ['yes', 'no'] + requirements: [ "urllib", "urllib2" ] author: Nandor Sivok ''' @@ -90,8 +98,6 @@ ansible host -m netscaler -a "nsc_host=nsc.example.com user=apiuser password=api import json -import urllib -import urllib2 import base64 import socket @@ -100,23 +106,25 @@ class netscaler(object): _nitro_base_url = '/nitro/v1/' + def __init__(self, module): + self.module = module + def http_request(self, api_endpoint, data_json={}): request_url = self._nsc_protocol + '://' + self._nsc_host + self._nitro_base_url + api_endpoint + data_json = urllib.urlencode(data_json) + if not len(data_json): + data_json = None - if len(data_json): - req = urllib2.Request(request_url, data_json) - req.add_header('Content-Type', 'application/x-www-form-urlencoded') - else: - req = urllib2.Request(request_url) + auth = base64.encodestring('%s:%s' % (self._nsc_user, self._nsc_pass)).replace('\n', '').strip() + headers = { + 'Authorization': 'Basic %s' % auth, + 'Content-Type' : 'application/x-www-form-urlencoded', + } - base64string = base64.encodestring('%s:%s' % (self._nsc_user, self._nsc_pass)).replace('\n', '').strip() - req.add_header('Authorization', "Basic %s" % base64string) + response, info = fetch_url(self.module, request_url, data=data_json, validate_certs=self.module.params['validate_certs']) - resp = urllib2.urlopen(req) - resp = json.load(resp) - - return resp + return json.load(response.read()) def prepare_request(self, action): resp = self.http_request( @@ -134,7 +142,7 @@ class netscaler(object): def core(module): - n = netscaler() + n = netscaler(module) n._nsc_host = module.params.get('nsc_host') n._nsc_user = module.params.get('user') n._nsc_pass = module.params.get('password') @@ -158,7 +166,8 @@ def main(): password = dict(required=True), action = dict(default='enable', choices=['enable','disable']), name = dict(default=socket.gethostname()), - type = dict(default='server', choices=['service', 'server']) + type = dict(default='server', choices=['service', 'server']), + validate_certs=dict(default='yes', type='bool'), ) ) @@ -177,4 +186,5 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * main() diff --git a/library/network/get_url b/library/network/get_url index 9704b8dbadb..c249c44049a 100644 --- a/library/network/get_url +++ b/library/network/get_url @@ -83,6 +83,13 @@ options: required: false default: 'yes' choices: ['yes', 'no'] + validate_certs: + description: + - If C(no), SSL certificates will not be validated. This should only be used + on personally controlled sites using self-signed certificates. + required: false + default: 'yes' + choices: ['yes', 'no'] others: description: - all arguments accepted by the M(file) module also work here @@ -108,19 +115,6 @@ try: except ImportError: HAS_HASHLIB=False -try: - import urllib2 - HAS_URLLIB2 = True -except ImportError: - HAS_URLLIB2 = False - -try: - import urlparse - import socket - HAS_URLPARSE = True -except ImportError: - HAS_URLPARSE=False - # ============================================================== # url handling @@ -130,80 +124,14 @@ def url_filename(url): return 'index.html' return fn -def url_do_get(module, url, dest, use_proxy, last_mod_time, force): - """ - Get url and return request and info - Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp - """ - - USERAGENT = 'ansible-httpget' - info = dict(url=url, dest=dest) - r = None - handlers = [] - - parsed = urlparse.urlparse(url) - - if '@' in parsed[1]: - credentials, netloc = parsed[1].split('@', 1) - if ':' in credentials: - username, password = credentials.split(':', 1) - else: - username = credentials - password = '' - parsed = list(parsed) - parsed[1] = netloc - - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - # this creates a password manager - passman.add_password(None, netloc, username, password) - # because we have put None at the start it will always - # use this username/password combination for urls - # for which `theurl` is a super-url - - authhandler = urllib2.HTTPBasicAuthHandler(passman) - # create the AuthHandler - handlers.append(authhandler) - - #reconstruct url without credentials - url = urlparse.urlunparse(parsed) - - if not use_proxy: - proxyhandler = urllib2.ProxyHandler({}) - handlers.append(proxyhandler) - - opener = urllib2.build_opener(*handlers) - urllib2.install_opener(opener) - request = urllib2.Request(url) - request.add_header('User-agent', USERAGENT) - - if last_mod_time and not force: - tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') - request.add_header('If-Modified-Since', tstamp) - else: - request.add_header('cache-control', 'no-cache') - - try: - r = urllib2.urlopen(request) - info.update(r.info()) - info['url'] = r.geturl() # The URL goes in too, because of redirects. - info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200)) - except urllib2.HTTPError, e: - # Must not fail_json() here so caller can handle HTTP 304 unmodified - info.update(dict(msg=str(e), status=e.code)) - except urllib2.URLError, e: - code = getattr(e, 'code', -1) - module.fail_json(msg="Request failed: %s" % str(e), status_code=code) - - return r, info - -def url_get(module, url, dest, use_proxy, last_mod_time, force): +def url_get(module, url, dest, use_proxy, last_mod_time, force, validate_certs): """ Download data from the url and store in a temporary file. Return (tempfile, info about the request) """ - req, info = url_do_get(module, url, dest, use_proxy, last_mod_time, force) + rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, validate_certs=validate_certs) if info['status'] == 304: module.exit_json(url=url, dest=dest, changed=False, msg=info.get('msg', '')) @@ -215,12 +143,12 @@ def url_get(module, url, dest, use_proxy, last_mod_time, force): fd, tempname = tempfile.mkstemp() f = os.fdopen(fd, 'wb') try: - shutil.copyfileobj(req, f) + shutil.copyfileobj(rsp, f) except Exception, err: os.remove(tempname) module.fail_json(msg="failed to create temporary content file: %s" % str(err)) f.close() - req.close() + rsp.close() return tempname, info def extract_filename_from_headers(headers): @@ -247,21 +175,15 @@ def extract_filename_from_headers(headers): def main(): - # does this really happen on non-ancient python? - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - if not HAS_URLPARSE: - module.fail_json(msg="urlparse is not installed") + argument_spec = url_argument_spec() + argument_spec.update( + dest = dict(required=True), + sha256sum = dict(default=''), + ) module = AnsibleModule( # not checking because of daisy chain to file module - argument_spec = dict( - url = dict(required=True), - dest = dict(required=True), - force = dict(default='no', aliases=['thirsty'], type='bool'), - sha256sum = dict(default=''), - use_proxy = dict(default='yes', type='bool') - ), + argument_spec = argument_spec, add_file_common_args=True ) @@ -270,6 +192,7 @@ def main(): force = module.params['force'] sha256sum = module.params['sha256sum'] use_proxy = module.params['use_proxy'] + validate_certs = module.params['validate_certs'] dest_is_dir = os.path.isdir(dest) last_mod_time = None @@ -284,7 +207,7 @@ def main(): last_mod_time = datetime.datetime.utcfromtimestamp(mtime) # download to tmpsrc - tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force) + tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, validate_certs) # Now the request has completed, we can finally generate the final # destination file name from the info dict. @@ -366,4 +289,5 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * main() diff --git a/library/notification/flowdock b/library/notification/flowdock index a5be40d1f10..32817d756dc 100644 --- a/library/notification/flowdock +++ b/library/notification/flowdock @@ -96,31 +96,12 @@ EXAMPLES = ''' tags=tag1,tag2,tag3 ''' -HAS_URLLIB = True -try: - import urllib -except ImportError: - HAS_URLLIB = False - -HAS_URLLIB2 = True -try: - import urllib2 -except ImportError: - HAS_URLLIB2 = False - - - # =========================================== # Module execution. # def main(): - if not HAS_URLLIB: - module.fail_json(msg="urllib is not installed") - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - module = AnsibleModule( argument_spec=dict( token=dict(required=True), @@ -187,14 +168,16 @@ def main(): module.exit_json(changed=False) # Send the data to Flowdock - try: - response = urllib2.urlopen(url, urllib.urlencode(params)) - except Exception, e: - module.fail_json(msg="unable to send msg: %s" % e) + data = urllib.urlencode(params) + response, info = fetch_url(module, url, data=data) + if info['status'] != 200: + module.fail_json(msg="unable to send msg: %s" % info['msg']) - module.exit_json(changed=False, msg=module.params["msg"]) + module.exit_json(changed=True, msg=module.params["msg"]) # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/notification/grove b/library/notification/grove index b759f025e29..1e2132cfb73 100644 --- a/library/notification/grove +++ b/library/notification/grove @@ -41,8 +41,6 @@ EXAMPLES = ''' message=deployed {{ target }} ''' -import urllib - BASE_URL = 'https://grove.io/api/notice/%s/' # ============================================================== @@ -57,7 +55,10 @@ def do_notify_grove(module, channel_token, service, message, url=None, icon_url= if icon_url is not None: my_data['icon_url'] = icon_url - urllib.urlopen(my_url, urllib.urlencode(my_data)) + data = urllib.urlencode(my_data) + response, info = fetch_url(module, my_url, data=data) + if info['status'] != 200: + module.fail_json(msg="failed to send notification: %s" % info['msg']) # ============================================================== # main diff --git a/library/notification/hipchat b/library/notification/hipchat index eec2b8c3618..c4b36d64ce7 100644 --- a/library/notification/hipchat +++ b/library/notification/hipchat @@ -60,22 +60,10 @@ EXAMPLES = ''' # HipChat module specific support methods. # -HAS_URLLIB = True -try: - import urllib -except ImportError: - HAS_URLLIB = False - -HAS_URLLIB2 = True -try: - import urllib2 -except ImportError: - HAS_URLLIB2 = False - MSG_URI = "https://api.hipchat.com/v1/rooms/message?" -def send_msg(token, room, msg_from, msg, msg_format='text', +def send_msg(module, token, room, msg_from, msg, msg_format='text', color='yellow', notify=False): '''sending message to hipchat''' @@ -92,8 +80,12 @@ def send_msg(token, room, msg_from, msg, msg_format='text', params['notify'] = 0 url = MSG_URI + "auth_token=%s" % (token) - response = urllib2.urlopen(url, urllib.urlencode(params)) - return response.read() + data = urllib.urlencode(params) + response, info = fetch_url(module, url, data=data) + if info['status'] == 200: + return response.read() + else: + module.fail_json(msg="failed to send message, return status=%s" % str(info['status'])) # =========================================== @@ -102,11 +94,6 @@ def send_msg(token, room, msg_from, msg, msg_format='text', def main(): - if not HAS_URLLIB: - module.fail_json(msg="urllib is not installed") - if not HAS_URLLIB2: - module.fail_json(msg="urllib2 is not installed") - module = AnsibleModule( argument_spec=dict( token=dict(required=True), @@ -130,15 +117,15 @@ def main(): notify = module.params["notify"] try: - send_msg(token, room, msg_from, msg, msg_format, - color, notify) + send_msg(module, token, room, msg_from, msg, msg_format, color, notify) except Exception, e: module.fail_json(msg="unable to sent msg: %s" % e) changed = True - module.exit_json(changed=changed, room=room, msg_from=msg_from, - msg=msg) + module.exit_json(changed=changed, room=room, msg_from=msg_from, msg=msg) # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main() diff --git a/library/packaging/apt_key b/library/packaging/apt_key index eee86337020..ff05bb93d1a 100644 --- a/library/packaging/apt_key +++ b/library/packaging/apt_key @@ -64,6 +64,14 @@ options: default: present description: - used to specify if key is being added or revoked + validate_certs: + description: + - If C(no), SSL certificates for the target url will not be validated. This should only be used + on personally controlled sites using self-signed certificates. + required: false + default: 'yes' + choices: ['yes', 'no'] + ''' EXAMPLES = ''' @@ -88,7 +96,6 @@ EXAMPLES = ''' # FIXME: standardize into module_common -from urllib2 import urlopen, URLError from traceback import format_exc from re import compile as re_compile # FIXME: standardize into module_common @@ -133,11 +140,8 @@ def download_key(module, url): if url is None: module.fail_json(msg="needed a URL but was not specified") try: - connection = urlopen(url) - if connection is None: - module.fail_json("error connecting to download key from url") - data = connection.read() - return data + rsp, info = fetch_url(module, url, validate_certs=module.params['validate_certs']) + return rsp.read() except Exception: module.fail_json(msg="error getting key id from url", traceback=format_exc()) @@ -175,7 +179,8 @@ def main(): file=dict(required=False), key=dict(required=False), keyring=dict(required=False), - state=dict(required=False, choices=['present', 'absent'], default='present') + state=dict(required=False, choices=['present', 'absent'], default='present'), + validate_certs=dict(default='yes', type='bool'), ), supports_check_mode=True ) @@ -240,4 +245,5 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * main() diff --git a/library/packaging/rpm_key b/library/packaging/rpm_key index 82532477348..9d85f30ac8b 100644 --- a/library/packaging/rpm_key +++ b/library/packaging/rpm_key @@ -42,6 +42,14 @@ options: choices: [present, absent] description: - Wheather the key will be imported or removed from the rpm db. + validate_certs: + description: + - If C(no) and the C(key) is a url starting with https, SSL certificates will not be validated. This should only be used + on personally controlled sites using self-signed certificates. + required: false + default: 'yes' + choices: ['yes', 'no'] + ''' EXAMPLES = ''' @@ -57,7 +65,6 @@ EXAMPLES = ''' import syslog import os.path import re -import urllib2 import tempfile # Attempt to download at most 8192 bytes. @@ -116,8 +123,8 @@ class RpmKey: def fetch_key(self, url, maxbytes=MAXBYTES): """Downloads a key from url, returns a valid path to a gpg key""" try: - fd = urllib2.urlopen(url) - key = fd.read(maxbytes) + rsp, info = fetch_url(self.module, url, validate_certs=self.module.params['validate_certs']) + key = rsp.read(maxbytes) if not is_pubkey(key): self.module.fail_json(msg="Not a public key: %s" % url) tmpfd, tmpname = tempfile.mkstemp() @@ -187,7 +194,8 @@ def main(): module = AnsibleModule( argument_spec = dict( state=dict(default='present', choices=['present', 'absent'], type='str'), - key=dict(required=True, type='str') + key=dict(required=True, type='str'), + validate_certs=dict(default='yes', type='bool'), ), supports_check_mode=True ) @@ -198,4 +206,5 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * main() diff --git a/library/source_control/github_hooks b/library/source_control/github_hooks index 55eb8d3c8d3..c5c5b648c7a 100644 --- a/library/source_control/github_hooks +++ b/library/source_control/github_hooks @@ -19,7 +19,6 @@ # along with Ansible. If not, see . import json -import urllib2 import base64 DOCUMENTATION = ''' @@ -51,6 +50,14 @@ options: - This tells the githooks module what you want it to do. required: true choices: [ "create", "cleanall" ] + validate_certs: + description: + - If C(no), SSL certificates for the target repo will not be validated. This should only be used + on personally controlled sites using self-signed certificates. + required: false + default: 'yes' + choices: ['yes', 'no'] + author: Phillip Gentry, CX Inc ''' @@ -62,16 +69,19 @@ EXAMPLES = ''' - local_action: github_hooks action=cleanall user={{ gituser }} oauthkey={{ oauthkey }} repo={{ repo }} ''' -def list(hookurl, oauthkey, repo, user): +def list(module, hookurl, oauthkey, repo, user): url = "%s/hooks" % repo auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '') - req = urllib2.Request(url) - req.add_header("Authorization", "Basic %s" % auth) - res = urllib2.urlopen(req) - out = res.read() - return False, out + headers = { + 'Authorization': 'Basic %s' % auth, + } + response, info = fetch_url(module, url, headers=headers, validate_certs=module.params['validate_certs']) + if info['status'] != 200: + return False, '' + else: + return False, response.read() -def clean504(hookurl, oauthkey, repo, user): +def clean504(module, hookurl, oauthkey, repo, user): current_hooks = list(hookurl, oauthkey, repo, user)[1] decoded = json.loads(current_hooks) @@ -79,11 +89,11 @@ def clean504(hookurl, oauthkey, repo, user): if hook['last_response']['code'] == 504: # print "Last response was an ERROR for hook:" # print hook['id'] - delete(hookurl, oauthkey, repo, user, hook['id']) + delete(module, hookurl, oauthkey, repo, user, hook['id']) return 0, current_hooks -def cleanall(hookurl, oauthkey, repo, user): +def cleanall(module, hookurl, oauthkey, repo, user): current_hooks = list(hookurl, oauthkey, repo, user)[1] decoded = json.loads(current_hooks) @@ -91,11 +101,11 @@ def cleanall(hookurl, oauthkey, repo, user): if hook['last_response']['code'] != 200: # print "Last response was an ERROR for hook:" # print hook['id'] - delete(hookurl, oauthkey, repo, user, hook['id']) + delete(module, hookurl, oauthkey, repo, user, hook['id']) return 0, current_hooks -def create(hookurl, oauthkey, repo, user): +def create(module, hookurl, oauthkey, repo, user): url = "%s/hooks" % repo values = { "active": True, @@ -107,29 +117,23 @@ def create(hookurl, oauthkey, repo, user): } data = json.dumps(values) auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '') - out='[]' - try : - req = urllib2.Request(url) - req.add_data(data) - req.add_header("Authorization", "Basic %s" % auth) - res = urllib2.urlopen(req) - out = res.read() - return 0, out - except urllib2.HTTPError, e : - if e.code == 422 : - return 0, out + headers = { + 'Authorization': 'Basic %s' % auth, + } + response, info = fetch_url(module, url, data=data, headers=headers, validate_certs=module.params['validate_certs']) + if info['status'] != 200: + return 0, '[]' + else: + return 0, response.read() -def delete(hookurl, oauthkey, repo, user, hookid): +def delete(module, hookurl, oauthkey, repo, user, hookid): url = "%s/hooks/%s" % (repo, hookid) auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '') - req = urllib2.Request(url) - req.get_method = lambda: 'DELETE' - req.add_header("Authorization", "Basic %s" % auth) - # req.add_header('Content-Type', 'application/xml') - # req.add_header('Accept', 'application/xml') - res = urllib2.urlopen(req) - out = res.read() - return out + headers = { + 'Authorization': 'Basic %s' % auth, + } + response, info = fetch_url(module, url, data=data, headers=headers, method='DELETE', validate_certs=module.params['validate_certs']) + return response.read() def main(): module = AnsibleModule( @@ -139,6 +143,7 @@ def main(): oauthkey=dict(required=True), repo=dict(required=True), user=dict(required=True), + validate_certs=dict(default='yes', type='bool'), ) ) @@ -149,16 +154,16 @@ def main(): user = module.params['user'] if action == "list": - (rc, out) = list(hookurl, oauthkey, repo, user) + (rc, out) = list(module, hookurl, oauthkey, repo, user) if action == "clean504": - (rc, out) = clean504(hookurl, oauthkey, repo, user) + (rc, out) = clean504(module, hookurl, oauthkey, repo, user) if action == "cleanall": - (rc, out) = cleanall(hookurl, oauthkey, repo, user) + (rc, out) = cleanall(module, hookurl, oauthkey, repo, user) if action == "create": - (rc, out) = create(hookurl, oauthkey, repo, user) + (rc, out) = create(module, hookurl, oauthkey, repo, user) if rc != 0: module.fail_json(msg="failed", result=out) @@ -168,4 +173,6 @@ def main(): # import module snippets from ansible.module_utils.basic import * +from ansible.module_utils.urls import * + main()