cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Split the fetch_url() function into fetch_url and open_url().

Browse source 
open_url() is suitable for use outside of a module environment.  Will
let us use open_url to do SSL cert verification in other, non-module
code.
This commit is contained in:
Toshio Kuratomi 2015-06-12 12:24:23 -07:00
parent 4a5a8ed963
commit 08c1ddd24e
1 changed files with 110 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

186
lib/ansible/module_utils/urls.py
View file

@ -26,12 +26,6 @@
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
try:
import urllib
HAS_URLLIB = True
except:
HAS_URLLIB = False
try:
import urllib2
HAS_URLLIB2 = True
@ -62,7 +56,9 @@ except ImportError:
import httplib
import os
import re
import sys
import socket
import platform
import tempfile
@ -89,6 +85,27 @@ zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg=
-----END CERTIFICATE-----
"""
#
# Exceptions
#
class ConnectionError(Exception):
"""Failed to connect to the server"""
pass
class ProxyError(ConnectionError):
"""Failure to connect because of a proxy"""
pass
class SSLValidationError(ConnectionError):
"""Failure to connect due to SSL validation failing"""
pass
class NoSSLError(SSLValidationError):
"""Needed to connect to an HTTPS url but no ssl library available to verify the certificate"""
pass
class CustomHTTPSConnection(httplib.HTTPSConnection):
def connect(self):
"Connect to a host on a given (SSL) port."
@ -153,7 +170,7 @@ def generic_urlparse(parts):
username, password = auth.split(':', 1)
generic_parts['username'] = username
generic_parts['password'] = password
generic_parts['hostname'] = hostnme
generic_parts['hostname'] = hostname
generic_parts['port'] = port
except:
generic_parts['username'] = None
@ -189,8 +206,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
'''
CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n"
def __init__(self, module, hostname, port):
self.module = module
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
@ -200,23 +216,22 @@ class SSLValidationHandler(urllib2.BaseHandler):
ca_certs = []
paths_checked = []
platform = get_platform()
distribution = get_distribution()
system = platform.system()
# build a list of paths to check for .crt/.pem files
# based on the platform type
paths_checked.append('/etc/ssl/certs')
if platform == 'Linux':
if system == 'Linux':
paths_checked.append('/etc/pki/ca-trust/extracted/pem')
paths_checked.append('/etc/pki/tls/certs')
paths_checked.append('/usr/share/ca-certificates/cacert.org')
elif platform == 'FreeBSD':
elif system == 'FreeBSD':
paths_checked.append('/usr/local/share/certs')
elif platform == 'OpenBSD':
elif system == 'OpenBSD':
paths_checked.append('/etc/ssl')
elif platform == 'NetBSD':
elif system == 'NetBSD':
ca_certs.append('/etc/openssl/certs')
elif platform == 'SunOS':
elif system == 'SunOS':
paths_checked.append('/opt/local/etc/openssl/certs')
# fall back to a user-deployed cert in a standard
@ -226,7 +241,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
tmp_fd, tmp_path = tempfile.mkstemp()
# Write the dummy ca cert if we are running on Mac OS X
if platform == 'Darwin':
if system == 'Darwin':
os.write(tmp_fd, DUMMY_CA_CERT)
# Default Homebrew path for OpenSSL certs
paths_checked.append('/usr/local/etc/openssl')
@ -259,7 +274,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
if int(resp_code) not in valid_codes:
raise Exception
except:
self.module.fail_json(msg='Connection to proxy failed')
raise ProxyError('Connection to proxy failed')
def detect_no_proxy(self, url):
'''
@ -304,7 +319,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
match_hostname(ssl_s.getpeercert(), self.hostname)
else:
self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
raise ProxyError('Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
else:
s.connect((self.hostname, self.port))
ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
@ -315,15 +330,14 @@ class SSLValidationHandler(urllib2.BaseHandler):
except (ssl.SSLError, socket.error), e:
# fail if we tried all of the certs but none worked
if 'connection refused' in str(e).lower():
self.module.fail_json(msg='Failed to connect to %s:%s.' % (self.hostname, self.port))
raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
else:
self.module.fail_json(
msg='Failed to validate the SSL certificate for %s:%s. ' % (self.hostname, self.port) + \
'Use validate_certs=no or make sure your managed systems have a valid CA certificate installed. ' + \
'Paths checked for this platform: %s' % ", ".join(paths_checked)
raise SSLValidationError('Failed to validate the SSL certificate for %s:%s. '
'Use validate_certs=False (insecure) or make sure your managed systems have a valid CA certificate installed. '
'Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked))
)
except CertificateError:
self.module.fail_json(msg="SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=no (insecure)" % self.hostname)
raise SSLValidationError("SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname)
try:
# cleanup the temp file created, don't worry
@ -336,55 +350,23 @@ class SSLValidationHandler(urllib2.BaseHandler):
https_request = http_request
def url_argument_spec():
'''
Creates an argument spec that can be used with any module
that will be requesting content via urllib/urllib2
'''
return dict(
url = dict(),
force = dict(default='no', aliases=['thirsty'], type='bool'),
http_agent = dict(default='ansible-httpget'),
use_proxy = dict(default='yes', type='bool'),
validate_certs = dict(default='yes', type='bool'),
url_username = dict(required=False),
url_password = dict(required=False),
)
def fetch_url(module, url, data=None, headers=None, method=None,
use_proxy=True, force=False, last_mod_time=None, timeout=10):
# Rewrite of fetch_url to not require the module environment
def open_url(url, data=None, headers=None, method=None, use_proxy=True,
force=False, last_mod_time=None, timeout=10, validate_certs=True,
url_username=None, url_password=None, http_agent=None):
'''
Fetches a file from an HTTP/FTP server using urllib2
'''
if not HAS_URLLIB:
module.fail_json(msg='urllib is not installed')
if not HAS_URLLIB2:
module.fail_json(msg='urllib2 is not installed')
elif not HAS_URLPARSE:
module.fail_json(msg='urlparse is not installed')
r = None
handlers = []
info = dict(url=url)
distribution = get_distribution()
# Get validate_certs from the module params
validate_certs = module.params.get('validate_certs', True)
# FIXME: change the following to use the generic_urlparse function
# to remove the indexed references for 'parsed'
parsed = urlparse.urlparse(url)
if parsed[0] == 'https' and validate_certs:
if not HAS_SSL:
if distribution == 'Redhat':
module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended. You can also install python-ssl from EPEL')
else:
module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended')
raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False, however this is unsafe and not recommended')
if not HAS_MATCH_HOSTNAME:
module.fail_json(msg='Available SSL validation does not check that the certificate matches the hostname. You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer. You could also use validate_certs=no, however this is unsafe and not recommended')
raise SSLValidationError('Available SSL validation does not check that the certificate matches the hostname. You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer. You could also use validate_certs=False, however this is unsafe and not recommended')
# do the cert validation
netloc = parsed[1]
@ -398,13 +380,14 @@ def fetch_url(module, url, data=None, headers=None, method=None,
port = 443
# create the SSL validation handler and
# add it to the list of handlers
ssl_handler = SSLValidationHandler(module, hostname, port)
ssl_handler = SSLValidationHandler(hostname, port)
handlers.append(ssl_handler)
if parsed[0] != 'ftp':
username = module.params.get('url_username', '')
username = url_username
if username:
password = module.params.get('url_password', '')
password = url_password
netloc = parsed[1]
elif '@' in parsed[1]:
credentials, netloc = parsed[1].split('@', 1)
@ -448,14 +431,14 @@ def fetch_url(module, url, data=None, headers=None, method=None,
if method:
if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'):
module.fail_json(msg='invalid HTTP request method; %s' % method.upper())
raise ConnectionError('invalid HTTP request method; %s' % method.upper())
request = RequestWithMethod(url, method.upper(), data)
else:
request = urllib2.Request(url, data)
# add the custom agent header, to help prevent issues
# with sites that block the default urllib agent string
request.add_header('User-agent', module.params.get('http_agent'))
request.add_header('User-agent', http_agent)
# if we're ok with getting a 304, set the timestamp in the
# header, otherwise make sure we don't get a cached copy
@ -468,20 +451,72 @@ def fetch_url(module, url, data=None, headers=None, method=None,
# user defined headers now, which may override things we've set above
if headers:
if not isinstance(headers, dict):
module.fail_json("headers provided to fetch_url() must be a dict")
raise ValueError("headers provided to fetch_url() must be a dict")
for header in headers:
request.add_header(header, headers[header])
if sys.version_info < (2,6,0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
r = urllib2.urlopen(request, None)
else:
r = urllib2.urlopen(request, None, timeout)
return r
#
# Module-related functions
#
def url_argument_spec():
'''
Creates an argument spec that can be used with any module
that will be requesting content via urllib/urllib2
'''
return dict(
url = dict(),
force = dict(default='no', aliases=['thirsty'], type='bool'),
http_agent = dict(default='ansible-httpget'),
use_proxy = dict(default='yes', type='bool'),
validate_certs = dict(default='yes', type='bool'),
url_username = dict(required=False),
url_password = dict(required=False),
)
def fetch_url(module, url, data=None, headers=None, method=None,
use_proxy=True, force=False, last_mod_time=None, timeout=10):
'''
Fetches a file from an HTTP/FTP server using urllib2. Requires the module environment
'''
if not HAS_URLLIB2:
module.fail_json(msg='urllib2 is not installed')
elif not HAS_URLPARSE:
module.fail_json(msg='urlparse is not installed')
# Get validate_certs from the module params
validate_certs = module.params.get('validate_certs', True)
username = module.params.get('url_username', '')
password = module.params.get('url_password', '')
http_agent = module.params.get('http_agent', None)
r = None
info = dict(url=url)
try:
if sys.version_info < (2,6,0):
# urlopen in python prior to 2.6.0 did not
# have a timeout parameter
r = urllib2.urlopen(request, None)
else:
r = urllib2.urlopen(request, None, timeout)
r = open_url(url, data=None, headers=None, method=None,
use_proxy=True, force=False, last_mod_time=None, timeout=10,
validate_certs=validate_certs, url_username=username,
url_password=password, http_agent=http_agent)
info.update(r.info())
info['url'] = r.geturl() # The URL goes in too, because of redirects.
info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
except NoSSLError, e:
distribution = get_distribution()
if distribution.lower() == 'redhat':
module.fail_json(msg='%s. You can also install python-ssl from EPEL' % str(e))
except (ConnectionError, ValueError), e:
module.fail_json(msg=str(e))
except urllib2.HTTPError, e:
info.update(dict(msg=str(e), status=e.code))
except urllib2.URLError, e:
@ -493,4 +528,3 @@ def fetch_url(module, url, data=None, headers=None, method=None,
info.update(dict(msg="An unknown error occurred: %s" % str(e), status=-1))
return r, info