Validate SSL certs accessed through urllib*

* Adds another module utility file which generalizes the
  access of urls via the urllib* libraries.
* Adds a new spec generator for common arguments.
* Makes the user-agent string configurable.

Fixes #6211

cloud/ec2_facts

@@ -41,7 +41,6 @@ EXAMPLES = '''
   when: ansible_ec2_instance_type == "t1.micro"
 '''
    
-import urllib2
 import socket
 import re
 
@@ -62,7 +61,8 @@ class Ec2Metadata(object):
                    'us-west-1',
                    'us-west-2')
 
-    def __init__(self, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None):
+    def __init__(self, module, ec2_metadata_uri=None, ec2_sshdata_uri=None, ec2_userdata_uri=None):
+        self.module   = module
         self.uri_meta = ec2_metadata_uri or self.ec2_metadata_uri
         self.uri_user = ec2_userdata_uri or self.ec2_userdata_uri
         self.uri_ssh  =  ec2_sshdata_uri or self.ec2_sshdata_uri
@@ -70,12 +70,9 @@ class Ec2Metadata(object):
         self._prefix   = 'ansible_ec2_%s'
 
     def _fetch(self, url):
-        try:
+        self.module.fail_json(msg="url is %s" % url)
-            return urllib2.urlopen(url).read()
+        (response, info) = fetch_url(self.module, url, force=True)
-        except urllib2.HTTPError:
+        return response.read()
-            return
-        except urllib2.URLError:
-            return
 
     def _mangle_fields(self, fields, uri, filter_patterns=['public-keys-0']):
         new_fields = {}
@@ -150,17 +147,20 @@ class Ec2Metadata(object):
         return data
 
 def main():
-
+    argument_spec = url_argument_spec()
-    ec2_facts = Ec2Metadata().run()
-    ec2_facts_result = dict(changed=False, ansible_facts=ec2_facts)
 
     module = AnsibleModule(
-        argument_spec = dict(),
+        argument_spec = argument_spec,
         supports_check_mode = True,
     )
+
+    ec2_facts = Ec2Metadata(module).run()
+    ec2_facts_result = dict(changed=False, ansible_facts=ec2_facts)
+
     module.exit_json(**ec2_facts_result)
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 
 main()

database/riak

@@ -138,22 +138,11 @@ def main():
     while True:
         if time.time() > timeout:
             module.fail_json(msg='Timeout, could not fetch Riak stats.')
-        try:
+        (response, info) = fetch_url(module, 'http://%s/stats' % (http_conn), force=True, timeout=5)
-            if sys.version_info<(2,6,0):
+        if info['status'] == 200:
-                stats_raw = urllib2.urlopen(
+            stats_raw = response.read()
-                    'http://%s/stats' % (http_conn), None).read()
-            else:
-                stats_raw = urllib2.urlopen(
-                    'http://%s/stats' % (http_conn), None, 5).read()
             break
-        except urllib2.HTTPError, e:
         time.sleep(5)
-        except urllib2.URLError, e:
-            time.sleep(5)
-        except socket.timeout:
-            time.sleep(5)
-        except Exception, e:
-            module.fail_json(msg='Could not fetch Riak stats: %s' % e)
 
     # here we attempt to load those stats,
     try:

monitoring/airbrake_deployment

@@ -52,6 +52,13 @@ options:
       - Optional URL to submit the notification to. Use to send notifications to Airbrake-compliant tools like Errbit.
     required: false
     default: https://airbrake.io/deploys
+  validate_certs:
+    description:
+      - If C(no), SSL certificates for the target url will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
 
 # informational: requirements for nodes
 requirements: [ urllib, urllib2 ]
@@ -64,29 +71,12 @@ EXAMPLES = '''
                        revision=4.2
 '''
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
 # ===========================================
 # Module execution.
 #
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -95,6 +85,7 @@ def main():
             repo=dict(required=False),
             revision=dict(required=False),
             url=dict(required=False, default='https://api.airbrake.io/deploys.txt')
+            validate_certs=dict(default='yes', type='bool'),
         ),
         supports_check_mode=True
     )
@@ -123,18 +114,16 @@ def main():
         module.exit_json(changed=True)
 
     # Send the data to airbrake
-    try:
+    data = urllib.urlencode(params)
-        req = urllib2.Request(url, urllib.urlencode(params))
+    response, info = fetch_url(module, url, data=data, validate_certs=module.params['validate_certs'])
-        result=urllib2.urlopen(req)
+    if info['status'] == 200:
-    except Exception, e:
-        module.fail_json(msg="unable to update airbrake via %s?%s : %s" % (url, urllib.urlencode(params), e))
-    else:
-        if result.code == 200:
         module.exit_json(changed=True)
     else:
-            module.fail_json(msg="HTTP result code: %d connecting to %s" % (result.code, url))
+        module.fail_json(msg="HTTP result code: %d connecting to %s" % (info['status'], url))
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
 

monitoring/boundary_meter

@@ -24,7 +24,6 @@ along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
 import datetime
-import urllib2
 import base64
 import os
 
@@ -74,12 +73,6 @@ EXAMPLES='''
 
 '''
 
-try:
-    import urllib2
-    HAS_URLLIB2 = True
-except ImportError:
-    HAS_URLLIB2 = False
-
 api_host = "api.boundary.com"
 config_directory = "/etc/bprobe"
 
@@ -101,7 +94,7 @@ def build_url(name, apiid, action, meter_id=None, cert_type=None):
     elif action == "delete":
         return  "https://%s/%s/meters/%s" % (api_host, apiid, meter_id)
 
-def http_request(name, apiid, apikey, action, meter_id=None, cert_type=None):
+def http_request(module, name, apiid, apikey, action, data=None, meter_id=None, cert_type=None):
     
     if meter_id is None:
         url = build_url(name, apiid, action)
@@ -111,11 +104,11 @@ def http_request(name, apiid, apikey, action, meter_id=None, cert_type=None):
         else:
             url = build_url(name, apiid, action, meter_id, cert_type)
 
-    auth = auth_encode(apikey)
+    headers = dict()
-    request = urllib2.Request(url)
+    headers["Authorization"] = "Basic %s" % auth_encode(apikey)
-    request.add_header("Authorization", "Basic %s" % (auth)) 
+    headers["Content-Type"] = "application/json"
-    request.add_header("Content-Type", "application/json")
+
-    return request
+    return fetch_url(module, url, data=data, headers=headers)
 
 def create_meter(module, name, apiid, apikey):
 
@@ -126,14 +119,10 @@ def create_meter(module, name, apiid, apikey):
         module.exit_json(status="Meter " + name + " already exists",changed=False)
     else:
         # If it doesn't exist, create it
-        request = http_request(name, apiid, apikey, action="create")
-        # A create request seems to need a json body with the name of the meter in it
         body = '{"name":"' + name + '"}'
-        request.add_data(body)
+        response, info = http_request(module, name, apiid, apikey, data=body, action="create")
 
-        try: 
+        if info['status'] != 200:
-            result = urllib2.urlopen(request)
-        except urllib2.URLError, e:
             module.fail_json(msg="Failed to connect to api host to create meter")
 
         # If the config directory doesn't exist, create it
@@ -160,15 +149,13 @@ def create_meter(module, name, apiid, apikey):
 
 def search_meter(module, name, apiid, apikey):
 
-    request = http_request(name, apiid, apikey, action="search")
+    response, info = http_request(module, name, apiid, apikey, action="search")
 
-    try: 
+    if info['status'] != 200:
-        result = urllib2.urlopen(request)
-    except urllib2.URLError, e:
         module.fail_json("Failed to connect to api host to search for meter")
 
     # Return meters
-    return json.loads(result.read())
+    return json.loads(response.read())
 
 def get_meter_id(module, name, apiid, apikey):
     # In order to delete the meter we need its id
@@ -186,16 +173,9 @@ def delete_meter(module, name, apiid, apikey):
     if meter_id is None:
         return 1, "Meter does not exist, so can't delete it"
     else:
-        action = "delete"
+        response, info = http_request(module, name, apiid, apikey, action, meter_id)
-        request = http_request(name, apiid, apikey, action, meter_id)
+        if info['status'] != 200:
-        # See http://stackoverflow.com/questions/4511598/how-to-make-http-delete-method-using-urllib2
+            module.fail_json("Failed to delete meter")
-        # urllib2 only does GET or POST I believe, but here we need delete
-        request.get_method = lambda: 'DELETE'
-
-        try: 
-            result = urllib2.urlopen(request)
-        except urllib2.URLError, e:
-            module.fail_json("Failed to connect to api host to delete meter")
 
         # Each new meter gets a new key.pem and ca.pem file, so they should be deleted
         types = ['cert', 'key']
@@ -214,17 +194,14 @@ def download_request(module, name, apiid, apikey, cert_type):
 
     if meter_id is not None:
         action = "certificates"
-        request = http_request(name, apiid, apikey, action, meter_id, cert_type)
+        response, info = http_request(module, name, apiid, apikey, action, meter_id, cert_type)
- 
+        if info['status'] != 200:
-        try:
-            result = urllib2.urlopen(request)
-        except urllib2.URLError, e:
             module.fail_json("Failed to connect to api host to download certificate")
 
         if result:
             try:
                 cert_file_path = '%s/%s.pem' % (config_directory,cert_type)
-                body = result.read()
+                body = response.read()
                 cert_file = open(cert_file_path, 'w')
                 cert_file.write(body)
                 cert_file.close
@@ -238,9 +215,6 @@ def download_request(module, name, apiid, apikey, cert_type):
 
 def main():
 
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
         state=dict(required=True, choices=['present', 'absent']),
@@ -268,5 +242,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()
 

monitoring/datadog_event

@@ -67,7 +67,6 @@ datadog_event: title="Testing from ansible" text="Test!"
 '''
 
 import socket
-from urllib2 import urlopen, Request, URLError
 
 def main():
     module = AnsibleModule(
@@ -97,8 +96,7 @@ def main():
     post_event(module)
 
 def post_event(module):
-    uri = "https://app.datadoghq.com/api/v1/events?api_key=" + \
+    uri = "https://app.datadoghq.com/api/v1/events?api_key=%s" % module.params['api_key']
-          module.params['api_key']
 
     body = dict(
         title=module.params['title'],
@@ -117,22 +115,20 @@ def post_event(module):
 
     json_body = module.jsonify(body)
     headers = {"Content-Type": "application/json"}
-    request = Request(uri, json_body, headers, unverifiable=True)
 
-    try:
+    (response, info) = fetch_url(module, uri, data=json_body, headers=headers)
-        response = urlopen(request)
+    if info['status'] == 200:
         response_body = response.read()
         response_json = module.from_json(response_body)
         if response_json['status'] == 'ok':
             module.exit_json(changed=True)
         else:
             module.fail_json(msg=response)
-
+    else:
-    except URLError, e:
+        module.fail_json(**info)
-        module.fail_json(msg="URL error: %s." % e)
-    except socket.error, e:
-        module.fail_json(msg="Socket error: %s to %s" % (e, uri))
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()

monitoring/newrelic_deployment

@@ -75,29 +75,12 @@ EXAMPLES = '''
                        revision=1.0
 '''
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
 # ===========================================
 # Module execution.
 #
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -134,29 +117,20 @@ def main():
         module.exit_json(changed=True)
 
     # Send the data to NewRelic
-    try:
+    url = "https://rpm.newrelic.com/deployments.xml"
-        req = urllib2.Request("https://rpm.newrelic.com/deployments.xml", urllib.urlencode(params))
+    data = urllib.urlencode(params)
-        req.add_header('x-api-key',module.params["token"])
+    headers = {
-        result=urllib2.urlopen(req)
+        'x-api-key': module.params["token"],
-        # urlopen behaves differently in python 2.4 and 2.6 so we handle
+    }
-        # both cases here.  In python 2.4 it throws an exception if the
+    response, info = fetch_url(module, url, data=data, headers=headers)
-        # return code is anything other than a 200.  In python 2.6 it
+    if info['status'] in (200, 201):
-        # doesn't throw an exception for any 2xx return codes.  In both
-        # cases we expect newrelic should return a 201 on success. So 
-        # to handle both cases, both the except & else cases below are
-        # effectively identical.
-    except Exception, e:
-        if e.code == 201:
         module.exit_json(changed=True)
     else:
-            module.fail_json(msg="unable to update newrelic: %s" % e)
+        module.fail_json(msg="unable to update newrelic: %s" % info['msg'])
-    else:
-        if result.code == 201:
-            module.exit_json(changed=True)
-        else:
-            module.fail_json(msg="result code: %d" % result.code)
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
 

monitoring/pagerduty

@@ -87,24 +87,23 @@ EXAMPLES='''
 
 import json
 import datetime
-import urllib2
 import base64
 
 
-def ongoing(name, user, passwd):
+def ongoing(module, name, user, passwd):
 
     url = "https://" + name + ".pagerduty.com/api/v1/maintenance_windows/ongoing"
     auth = base64.encodestring('%s:%s' % (user, passwd)).replace('\n', '')
+    headers = {"Authorization": "Basic %s" % auth}
 
-    req = urllib2.Request(url)
+    response, info = fetch_url(module, url, headers=headers)
-    req.add_header("Authorization", "Basic %s" % auth)
+    if info['status'] != 200:
-    res = urllib2.urlopen(req)
+        module.fail_json(msg="failed to lookup the ongoing window: %s" % info['msg'])
-    out = res.read()
 
-    return False, out
+    return False, response.read()
 
 
-def create(name, user, passwd, service, hours, desc):
+def create(module, name, user, passwd, service, hours, desc):
 
     now = datetime.datetime.utcnow()
     later = now + datetime.timedelta(hours=int(hours))
@@ -113,15 +112,17 @@ def create(name, user, passwd, service, hours, desc):
 
     url = "https://" + name + ".pagerduty.com/api/v1/maintenance_windows"
     auth = base64.encodestring('%s:%s' % (user, passwd)).replace('\n', '')
+    headers = {
+        'Authorization': 'Basic %s' % auth,
+        'Content-Type' : 'application/json',
+    }
     data = json.dumps({'maintenance_window': {'start_time': start, 'end_time': end, 'description': desc, 'service_ids': [service]}})
 
-    req = urllib2.Request(url, data)
+    response, info = fetch_url(module, url, data=data, headers=headers, method='POST')
-    req.add_header("Authorization", "Basic %s" % auth)
+    if info['status'] != 200:
-    req.add_header('Content-Type', 'application/json')
+        module.fail_json(msg="failed to create the window: %s" % info['msg'])
-    res = urllib2.urlopen(req)
-    out = res.read()
 
-    return False, out
+    return False, response.read()
 
 
 def main():
@@ -149,10 +150,10 @@ def main():
     if state == "running" or state == "started":
         if not service:
             module.fail_json(msg="service not specified")
-        (rc, out) = create(name, user, passwd, service, hours, desc)
+        (rc, out) = create(module, name, user, passwd, service, hours, desc)
 
     if state == "ongoing":
-        (rc, out) = ongoing(name, user, passwd)
+        (rc, out) = ongoing(module, name, user, passwd)
 
     if rc != 0:
         module.fail_json(msg="failed", result=out)
@@ -161,4 +162,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()

net_infrastructure/dnsmadeeasy

@@ -106,8 +106,6 @@ EXAMPLES = '''
 
 IMPORT_ERROR = None
 try:
-    import urllib
-    import urllib2
     import json
     from time import strftime, gmtime
     import hashlib
@@ -115,22 +113,6 @@ try:
 except ImportError, e:
     IMPORT_ERROR = str(e)
 
-
-class RequestWithMethod(urllib2.Request):
-
-    """Workaround for using DELETE/PUT/etc with urllib2"""
-
-    def __init__(self, url, method, data=None, headers={}):
-        self._method = method
-        urllib2.Request.__init__(self, url, data, headers)
-
-    def get_method(self):
-        if self._method:
-            return self._method
-        else:
-            return urllib2.Request.get_method(self)
-
-
 class DME2:
 
     def __init__(self, apikey, secret, domain, module):
@@ -169,16 +151,10 @@ class DME2:
         url = self.baseurl + resource
         if data and not isinstance(data, basestring):
             data = urllib.urlencode(data)
-        request = RequestWithMethod(url, method, data, self._headers())
 
-        try:
+        response, info = fetch_url(self.module, url, data=data, method=method)
-            response = urllib2.urlopen(request)
+        if info['status'] not in (200, 201, 204):
-        except urllib2.HTTPError, e:
+            self.module.fail_json(msg="%s returned %s, with body: %s" % (url, info['status'], info['msg']))
-            self.module.fail_json(
-                msg="%s returned %s, with body: %s" % (url, e.code, e.read()))
-        except Exception, e:
-            self.module.fail_json(
-                msg="Failed contacting: %s : Exception %s" % (url, e.message()))
 
         try:
             return json.load(response)
@@ -338,4 +314,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()

net_infrastructure/netscaler

@@ -73,6 +73,14 @@ options:
     default: server
     choices: ["server", "service"]
     aliases: []
+  validate_certs:
+    description:
+      - If C(no), SSL certificates for the target url will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
+
 requirements: [ "urllib", "urllib2" ]
 author: Nandor Sivok
 '''
@@ -90,8 +98,6 @@ ansible host -m netscaler -a "nsc_host=nsc.example.com user=apiuser password=api
 
 
 import json
-import urllib
-import urllib2
 import base64
 import socket
 
@@ -100,23 +106,25 @@ class netscaler(object):
 
     _nitro_base_url = '/nitro/v1/'
 
+    def __init__(self, module):
+        self.module = module
+
     def http_request(self, api_endpoint, data_json={}):
         request_url = self._nsc_protocol + '://' + self._nsc_host + self._nitro_base_url + api_endpoint
+
         data_json = urllib.urlencode(data_json)
+        if not len(data_json):
+            data_json = None
 
-        if len(data_json):
+        auth = base64.encodestring('%s:%s' % (self._nsc_user, self._nsc_pass)).replace('\n', '').strip()
-            req = urllib2.Request(request_url, data_json)
+        headers = {
-            req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+            'Authorization': 'Basic %s' % auth,
-        else:
+            'Content-Type' : 'application/x-www-form-urlencoded',
-            req = urllib2.Request(request_url)
+        }
 
-        base64string = base64.encodestring('%s:%s' % (self._nsc_user, self._nsc_pass)).replace('\n', '').strip()
+        response, info = fetch_url(self.module, request_url, data=data_json, validate_certs=self.module.params['validate_certs'])
-        req.add_header('Authorization', "Basic %s" % base64string)
 
-        resp = urllib2.urlopen(req)
+        return json.load(response.read())
-        resp = json.load(resp)
-
-        return resp
 
     def prepare_request(self, action):
         resp = self.http_request(
@@ -134,7 +142,7 @@ class netscaler(object):
 
 
 def core(module):
-    n = netscaler()
+    n = netscaler(module)
     n._nsc_host = module.params.get('nsc_host')
     n._nsc_user = module.params.get('user')
     n._nsc_pass = module.params.get('password')
@@ -158,7 +166,8 @@ def main():
             password = dict(required=True),
             action = dict(default='enable', choices=['enable','disable']),
             name = dict(default=socket.gethostname()),
-            type = dict(default='server', choices=['service', 'server'])
+            type = dict(default='server', choices=['service', 'server']),
+            validate_certs=dict(default='yes', type='bool'),
         )
     )
 
@@ -177,4 +186,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()

network/get_url

@@ -83,6 +83,13 @@ options:
     required: false
     default: 'yes'
     choices: ['yes', 'no']
+  validate_certs:
+    description:
+      - If C(no), SSL certificates will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
   others:
     description:
       - all arguments accepted by the M(file) module also work here
@@ -108,19 +115,6 @@ try:
 except ImportError:
     HAS_HASHLIB=False
 
-try:
-    import urllib2
-    HAS_URLLIB2 = True
-except ImportError:
-    HAS_URLLIB2 = False
-
-try:
-    import urlparse
-    import socket
-    HAS_URLPARSE = True
-except ImportError:
-    HAS_URLPARSE=False
-
 # ==============================================================
 # url handling
 
@@ -130,80 +124,14 @@ def url_filename(url):
         return 'index.html'
     return fn
 
-def url_do_get(module, url, dest, use_proxy, last_mod_time, force):
+def url_get(module, url, dest, use_proxy, last_mod_time, force, validate_certs):
-    """
-    Get url and return request and info
-    Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp
-    """
-
-    USERAGENT = 'ansible-httpget'
-    info = dict(url=url, dest=dest)
-    r = None
-    handlers = []
-
-    parsed = urlparse.urlparse(url)
-
-    if '@' in parsed[1]:
-        credentials, netloc = parsed[1].split('@', 1)
-        if ':' in credentials:
-            username, password = credentials.split(':', 1)
-        else:
-            username = credentials
-            password = ''
-        parsed = list(parsed)
-        parsed[1] = netloc
-
-        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-        # this creates a password manager
-        passman.add_password(None, netloc, username, password)
-        # because we have put None at the start it will always
-        # use this username/password combination for  urls
-        # for which `theurl` is a super-url
-
-        authhandler = urllib2.HTTPBasicAuthHandler(passman)
-        # create the AuthHandler
-        handlers.append(authhandler)
-
-        #reconstruct url without credentials
-        url = urlparse.urlunparse(parsed)
-
-    if not use_proxy:
-        proxyhandler = urllib2.ProxyHandler({})
-        handlers.append(proxyhandler)
-
-    opener = urllib2.build_opener(*handlers)
-    urllib2.install_opener(opener)
-    request = urllib2.Request(url)
-    request.add_header('User-agent', USERAGENT)
-
-    if last_mod_time and not force:
-        tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000')
-        request.add_header('If-Modified-Since', tstamp)
-    else:
-        request.add_header('cache-control', 'no-cache')
-
-    try:
-        r = urllib2.urlopen(request)
-        info.update(r.info())
-        info['url'] = r.geturl()  # The URL goes in too, because of redirects.
-        info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
-    except urllib2.HTTPError, e:
-        # Must not fail_json() here so caller can handle HTTP 304 unmodified
-        info.update(dict(msg=str(e), status=e.code))
-    except urllib2.URLError, e:
-        code = getattr(e, 'code', -1)
-        module.fail_json(msg="Request failed: %s" % str(e), status_code=code)
-
-    return r, info
-
-def url_get(module, url, dest, use_proxy, last_mod_time, force):
     """
     Download data from the url and store in a temporary file.
 
     Return (tempfile, info about the request)
     """
 
-    req, info = url_do_get(module, url, dest, use_proxy, last_mod_time, force)
+    rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, validate_certs=validate_certs)
 
     if info['status'] == 304:
         module.exit_json(url=url, dest=dest, changed=False, msg=info.get('msg', ''))
@@ -215,12 +143,12 @@ def url_get(module, url, dest, use_proxy, last_mod_time, force):
     fd, tempname = tempfile.mkstemp()
     f = os.fdopen(fd, 'wb')
     try:
-        shutil.copyfileobj(req, f)
+        shutil.copyfileobj(rsp, f)
     except Exception, err:
         os.remove(tempname)
         module.fail_json(msg="failed to create temporary content file: %s" % str(err))
     f.close()
-    req.close()
+    rsp.close()
     return tempname, info
 
 def extract_filename_from_headers(headers):
@@ -247,21 +175,15 @@ def extract_filename_from_headers(headers):
 
 def main():
 
-    # does this really happen on non-ancient python?
+    argument_spec = url_argument_spec()
-    if not HAS_URLLIB2:
+    argument_spec.update(
-        module.fail_json(msg="urllib2 is not installed")
+        dest = dict(required=True),
-    if not HAS_URLPARSE:
+        sha256sum = dict(default=''),
-        module.fail_json(msg="urlparse is not installed")
+    )
 
     module = AnsibleModule(
         # not checking because of daisy chain to file module
-        argument_spec = dict(
+        argument_spec = argument_spec,
-            url = dict(required=True),
-            dest = dict(required=True),
-            force = dict(default='no', aliases=['thirsty'], type='bool'),
-            sha256sum = dict(default=''),
-            use_proxy = dict(default='yes', type='bool')
-        ),
         add_file_common_args=True
     )
 
@@ -270,6 +192,7 @@ def main():
     force = module.params['force']
     sha256sum = module.params['sha256sum']
     use_proxy = module.params['use_proxy']
+    validate_certs = module.params['validate_certs']
 
     dest_is_dir = os.path.isdir(dest)
     last_mod_time = None
@@ -284,7 +207,7 @@ def main():
         last_mod_time = datetime.datetime.utcfromtimestamp(mtime)
 
     # download to tmpsrc
-    tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force)
+    tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, validate_certs)
 
     # Now the request has completed, we can finally generate the final
     # destination file name from the info dict.
@@ -366,4 +289,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()

notification/flowdock

@@ -96,31 +96,12 @@ EXAMPLES = '''
             tags=tag1,tag2,tag3
 '''
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
-
-
 # ===========================================
 # Module execution.
 #
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -187,14 +168,16 @@ def main():
         module.exit_json(changed=False)
 
     # Send the data to Flowdock
-    try:
+    data = urllib.urlencode(params)
-        response = urllib2.urlopen(url, urllib.urlencode(params))
+    response, info = fetch_url(module, url, data=data)
-    except Exception, e:
+    if info['status'] != 200:
-        module.fail_json(msg="unable to send msg: %s" % e)
+        module.fail_json(msg="unable to send msg: %s" % info['msg'])
 
-    module.exit_json(changed=False, msg=module.params["msg"])
+    module.exit_json(changed=True, msg=module.params["msg"])
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()
 

notification/grove

@@ -41,8 +41,6 @@ EXAMPLES = '''
     message=deployed {{ target }}
 '''
 
-import urllib
-
 BASE_URL = 'https://grove.io/api/notice/%s/'
 
 # ==============================================================
@@ -57,7 +55,10 @@ def do_notify_grove(module, channel_token, service, message, url=None, icon_url=
     if icon_url is not None:
         my_data['icon_url'] = icon_url
 
-    urllib.urlopen(my_url, urllib.urlencode(my_data))
+    data = urllib.urlencode(my_data)
+    response, info = fetch_url(module, my_url, data=data)
+    if info['status'] != 200:
+        module.fail_json(msg="failed to send notification: %s" % info['msg'])
 
 # ==============================================================
 # main

notification/hipchat

@@ -60,22 +60,10 @@ EXAMPLES = '''
 # HipChat module specific support methods.
 #
 
-HAS_URLLIB = True
-try:
-    import urllib
-except ImportError:
-    HAS_URLLIB = False
-
-HAS_URLLIB2 = True
-try:
-    import urllib2
-except ImportError:
-    HAS_URLLIB2 = False
-
 MSG_URI = "https://api.hipchat.com/v1/rooms/message?"
 
 
-def send_msg(token, room, msg_from, msg, msg_format='text',
+def send_msg(module, token, room, msg_from, msg, msg_format='text',
              color='yellow', notify=False):
     '''sending message to hipchat'''
 
@@ -92,8 +80,12 @@ def send_msg(token, room, msg_from, msg, msg_format='text',
         params['notify'] = 0
 
     url = MSG_URI + "auth_token=%s" % (token)
-    response = urllib2.urlopen(url, urllib.urlencode(params))
+    data = urllib.urlencode(params)
+    response, info = fetch_url(module, url, data=data)
+    if info['status'] == 200:
         return response.read()
+    else:
+        module.fail_json(msg="failed to send message, return status=%s" % str(info['status']))
 
 
 # ===========================================
@@ -102,11 +94,6 @@ def send_msg(token, room, msg_from, msg, msg_format='text',
 
 def main():
 
-    if not HAS_URLLIB:
-        module.fail_json(msg="urllib is not installed")
-    if not HAS_URLLIB2:
-        module.fail_json(msg="urllib2 is not installed")
-
     module = AnsibleModule(
         argument_spec=dict(
             token=dict(required=True),
@@ -130,15 +117,15 @@ def main():
     notify = module.params["notify"]
 
     try:
-        send_msg(token, room, msg_from, msg, msg_format,
+        send_msg(module, token, room, msg_from, msg, msg_format, color, notify)
-                 color, notify)
     except Exception, e:
         module.fail_json(msg="unable to sent msg: %s" % e)
 
     changed = True
-    module.exit_json(changed=changed, room=room, msg_from=msg_from,
+    module.exit_json(changed=changed, room=room, msg_from=msg_from, msg=msg)
-                     msg=msg)
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()

packaging/apt_key

@@ -64,6 +64,14 @@ options:
         default: present
         description:
             - used to specify if key is being added or revoked
+    validate_certs:
+        description:
+            - If C(no), SSL certificates for the target url will not be validated. This should only be used
+              on personally controlled sites using self-signed certificates.
+        required: false
+        default: 'yes'
+        choices: ['yes', 'no']
+
 '''
 
 EXAMPLES = '''
@@ -88,7 +96,6 @@ EXAMPLES = '''
 
 
 # FIXME: standardize into module_common
-from urllib2 import urlopen, URLError
 from traceback import format_exc
 from re import compile as re_compile
 # FIXME: standardize into module_common
@@ -133,11 +140,8 @@ def download_key(module, url):
     if url is None:
         module.fail_json(msg="needed a URL but was not specified")
     try:
-        connection = urlopen(url)
+        rsp, info = fetch_url(module, url, validate_certs=module.params['validate_certs'])
-        if connection is None:
+        return rsp.read()
-            module.fail_json("error connecting to download key from url")
-        data = connection.read()
-        return data
     except Exception:
         module.fail_json(msg="error getting key id from url", traceback=format_exc())
 
@@ -175,7 +179,8 @@ def main():
             file=dict(required=False),
             key=dict(required=False),
             keyring=dict(required=False),
-            state=dict(required=False, choices=['present', 'absent'], default='present')
+            state=dict(required=False, choices=['present', 'absent'], default='present'),
+            validate_certs=dict(default='yes', type='bool'),
         ),
         supports_check_mode=True
     )
@@ -240,4 +245,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()

packaging/rpm_key

@@ -42,6 +42,14 @@ options:
       choices: [present, absent]
       description:
           - Wheather the key will be imported or removed from the rpm db.
+    validate_certs:
+      description:
+          - If C(no) and the C(key) is a url starting with https, SSL certificates will not be validated. This should only be used
+            on personally controlled sites using self-signed certificates.
+      required: false
+      default: 'yes'
+      choices: ['yes', 'no']
+
 '''
 
 EXAMPLES = '''
@@ -57,7 +65,6 @@ EXAMPLES = '''
 import syslog
 import os.path
 import re
-import urllib2
 import tempfile
 
 # Attempt to download at most 8192 bytes.
@@ -116,8 +123,8 @@ class RpmKey:
     def fetch_key(self, url, maxbytes=MAXBYTES):
         """Downloads a key from url, returns a valid path to a gpg key"""
         try:
-            fd = urllib2.urlopen(url)
+            rsp, info = fetch_url(self.module, url, validate_certs=self.module.params['validate_certs'])
-            key = fd.read(maxbytes)
+            key = rsp.read(maxbytes)
             if not is_pubkey(key):
                 self.module.fail_json(msg="Not a public key: %s" % url)
             tmpfd, tmpname = tempfile.mkstemp()
@@ -187,7 +194,8 @@ def main():
     module = AnsibleModule(
             argument_spec = dict(
                 state=dict(default='present', choices=['present', 'absent'], type='str'),
-                key=dict(required=True, type='str')
+                key=dict(required=True, type='str'),
+                validate_certs=dict(default='yes', type='bool'),
                 ),
             supports_check_mode=True
             )
@@ -198,4 +206,5 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
 main()

source_control/github_hooks

@@ -19,7 +19,6 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
-import urllib2
 import base64
 
 DOCUMENTATION = '''
@@ -51,6 +50,14 @@ options:
       - This tells the githooks module what you want it to do.
     required: true
     choices: [ "create", "cleanall" ]
+  validate_certs:
+    description:
+      - If C(no), SSL certificates for the target repo will not be validated. This should only be used
+        on personally controlled sites using self-signed certificates.
+    required: false
+    default: 'yes'
+    choices: ['yes', 'no']
+
 author: Phillip Gentry, CX Inc
 '''
 
@@ -62,16 +69,19 @@ EXAMPLES = '''
 - local_action: github_hooks action=cleanall user={{ gituser }} oauthkey={{ oauthkey }} repo={{ repo }}
 '''
 
-def list(hookurl, oauthkey, repo, user):
+def list(module, hookurl, oauthkey, repo, user):
     url = "%s/hooks" % repo
     auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '')
-    req = urllib2.Request(url)
+    headers = {
-    req.add_header("Authorization", "Basic %s" % auth)
+        'Authorization': 'Basic %s' % auth,
-    res = urllib2.urlopen(req)
+    }
-    out = res.read()
+    response, info = fetch_url(module, url, headers=headers, validate_certs=module.params['validate_certs'])
-    return False, out
+    if info['status'] != 200:
+        return False, ''
+    else:
+        return False, response.read()
 
-def clean504(hookurl, oauthkey, repo, user):
+def clean504(module, hookurl, oauthkey, repo, user):
     current_hooks = list(hookurl, oauthkey, repo, user)[1]
     decoded = json.loads(current_hooks)
 
@@ -79,11 +89,11 @@ def clean504(hookurl, oauthkey, repo, user):
         if hook['last_response']['code'] == 504:
             # print "Last response was an ERROR for hook:"
             # print hook['id']
-            delete(hookurl, oauthkey, repo, user, hook['id'])
+            delete(module, hookurl, oauthkey, repo, user, hook['id'])
             
     return 0, current_hooks
 
-def cleanall(hookurl, oauthkey, repo, user):
+def cleanall(module, hookurl, oauthkey, repo, user):
     current_hooks = list(hookurl, oauthkey, repo, user)[1]
     decoded = json.loads(current_hooks)
 
@@ -91,11 +101,11 @@ def cleanall(hookurl, oauthkey, repo, user):
         if hook['last_response']['code'] != 200:
             # print "Last response was an ERROR for hook:"
             # print hook['id']
-            delete(hookurl, oauthkey, repo, user, hook['id'])
+            delete(module, hookurl, oauthkey, repo, user, hook['id'])
             
     return 0, current_hooks
 
-def create(hookurl, oauthkey, repo, user):
+def create(module, hookurl, oauthkey, repo, user):
     url = "%s/hooks" % repo
     values = {
         "active": True,
@@ -107,29 +117,23 @@ def create(hookurl, oauthkey, repo, user):
         }
     data = json.dumps(values)
     auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '')
-    out='[]'
+    headers = {
-    try :
+        'Authorization': 'Basic %s' % auth,
-        req = urllib2.Request(url)
+    }
-        req.add_data(data)
+    response, info = fetch_url(module, url, data=data, headers=headers, validate_certs=module.params['validate_certs'])
-        req.add_header("Authorization", "Basic %s" % auth)
+    if info['status'] != 200:
-        res = urllib2.urlopen(req)
+        return 0, '[]'
-        out = res.read()
+    else:
-        return 0, out
+        return 0, response.read()
-    except urllib2.HTTPError, e :
-        if e.code == 422 :
-            return 0, out
 
-def delete(hookurl, oauthkey, repo, user, hookid):
+def delete(module, hookurl, oauthkey, repo, user, hookid):
     url = "%s/hooks/%s" % (repo, hookid)
     auth = base64.encodestring('%s:%s' % (user, oauthkey)).replace('\n', '')
-    req = urllib2.Request(url)
+    headers = {
-    req.get_method = lambda: 'DELETE'
+        'Authorization': 'Basic %s' % auth,
-    req.add_header("Authorization", "Basic %s" % auth)
+    }
-    # req.add_header('Content-Type', 'application/xml') 
+    response, info = fetch_url(module, url, data=data, headers=headers, method='DELETE', validate_certs=module.params['validate_certs'])
-    # req.add_header('Accept', 'application/xml')
+    return response.read()
-    res = urllib2.urlopen(req)
-    out = res.read()
-    return out
 
 def main():
     module = AnsibleModule(
@@ -139,6 +143,7 @@ def main():
         oauthkey=dict(required=True),
         repo=dict(required=True),
         user=dict(required=True),
+        validate_certs=dict(default='yes', type='bool'),
         )
     )
 
@@ -149,16 +154,16 @@ def main():
     user = module.params['user']
 
     if action == "list":
-        (rc, out) = list(hookurl, oauthkey, repo, user)
+        (rc, out) = list(module, hookurl, oauthkey, repo, user)
 
     if action == "clean504":
-        (rc, out) = clean504(hookurl, oauthkey, repo, user)
+        (rc, out) = clean504(module, hookurl, oauthkey, repo, user)
 
     if action == "cleanall":
-        (rc, out) = cleanall(hookurl, oauthkey, repo, user)
+        (rc, out) = cleanall(module, hookurl, oauthkey, repo, user)
 
     if action == "create":
-        (rc, out) = create(hookurl, oauthkey, repo, user)
+        (rc, out) = create(module, hookurl, oauthkey, repo, user)
 
     if rc != 0:
         module.fail_json(msg="failed", result=out)
@@ -168,4 +173,6 @@ def main():
 
 # import module snippets
 from ansible.module_utils.basic import *
+from ansible.module_utils.urls import *
+
 main()