cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #2973 from sivel/uri-no-httplib2

Browse source 
Drop dependency on httplib2 in the uri module
This commit is contained in:
Matt Martz 2016-02-23 11:53:33 -06:00
commit e575014cdd
1 changed files with 115 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

223
network/basics/uri.py
View file

@ -23,9 +23,7 @@
import cgi
import shutil
import tempfile
import base64
import datetime
from distutils.version import LooseVersion
try:
import json
@ -48,7 +46,8 @@ options:
default: null
dest:
description:
- path of where to download the file to (if desired). If I(dest) is a directory, the basename of the file on the remote server will be used.
- path of where to download the file to (if desired). If I(dest) is a
directory, the basename of the file on the remote server will be used.
required: false
default: null
user:
@ -63,12 +62,15 @@ options:
default: null
body:
description:
- The body of the http request/response to the web service. If C(body_format) is set to 'json' it will take an already formated JSON string or convert a data structure into JSON.
- The body of the http request/response to the web service. If C(body_format) is set
to 'json' it will take an already formated JSON string or convert a data structure
into JSON.
required: false
default: null
body_format:
description:
- The serialization format of the body. When set to json, encodes the body argument, if needed, and automatically sets the Content-Type header accordingly.
- The serialization format of the body. When set to json, encodes the
body argument, if needed, and automatically sets the Content-Type header accordingly.
required: false
choices: [ "raw", "json" ]
default: raw
@ -81,13 +83,16 @@ options:
default: "GET"
return_content:
description:
- Whether or not to return the body of the request as a "content" key in the dictionary result. If the reported Content-type is "application/json", then the JSON is additionally loaded into a key called C(json) in the dictionary results.
- Whether or not to return the body of the request as a "content" key in
the dictionary result. If the reported Content-type is
"application/json", then the JSON is additionally loaded into a key
called C(json) in the dictionary results.
required: false
choices: [ "yes", "no" ]
default: "no"
force_basic_auth:
description:
- httplib2, the library used by the uri module only sends authentication information when a webservice
- The library used by the uri module only sends authentication information when a webservice
responds to an initial request with a 401 status. Since some basic auth services do not properly
send a 401, logins will fail. This option forces the sending of the Basic authentication header
upon initial request.
@ -115,7 +120,8 @@ options:
required: false
status_code:
description:
- A valid, numeric, HTTP status code that signifies success of the request. Can also be comma separated list of status codes.
- A valid, numeric, HTTP status code that signifies success of the
request. Can also be comma separated list of status codes.
required: false
default: 200
timeout:
@ -128,8 +134,16 @@ options:
- Any parameter starting with "HEADER_" is a sent with your request as a header.
For example, HEADER_Content-Type="application/json" would send the header
"Content-Type" along with your request with a value of "application/json".
This option is deprecated as of C(2.1) and may be removed in a future
release. Use I(headers) instead.
required: false
default: null
headers:
description:
- Add custom HTTP headers to a request in the format of a YAML hash
required: false
default: null
version_added: '2.1'
others:
description:
- all arguments accepted by the M(file) module also work here
@ -143,10 +157,8 @@ options:
default: 'yes'
choices: ['yes', 'no']
version_added: '1.9.2'
# informational: requirements for nodes
requirements:
- httplib2 >= 0.7.0
notes:
- The dependency on httplib2 was removed in Ansible 2.1
author: "Romeo Theriault (@romeotheriault)"
'''
@ -154,7 +166,8 @@ EXAMPLES = '''
# Check that you can connect (GET) to a page and it returns a status 200
- uri: url=http://www.example.com
# Check that a page returns a status 200 and fail if the word AWESOME is not in the page contents.
# Check that a page returns a status 200 and fail if the word AWESOME is not
# in the page contents.
- action: uri url=http://www.example.com return_content=yes
register: webpage
@ -201,23 +214,6 @@ EXAMPLES = '''
'''
HAS_HTTPLIB2 = False
try:
import httplib2
if LooseVersion(httplib2.__version__) >= LooseVersion('0.7'):
HAS_HTTPLIB2 = True
except ImportError, AttributeError:
# AttributeError if __version__ is not present
pass
HAS_URLPARSE = True
try:
import urlparse
import socket
except ImportError:
HAS_URLPARSE = False
def write_file(module, url, dest, content):
# create a tempfile with some test content
@ -247,15 +243,15 @@ def write_file(module, url, dest, content):
# raise an error if copy has no permission on dest
if not os.access(dest, os.W_OK):
os.remove(tmpsrc)
module.fail_json( msg="Destination %s not writable" % (dest))
module.fail_json(msg="Destination %s not writable" % (dest))
if not os.access(dest, os.R_OK):
os.remove(tmpsrc)
module.fail_json( msg="Destination %s not readable" % (dest))
module.fail_json(msg="Destination %s not readable" % (dest))
checksum_dest = module.sha1(dest)
else:
if not os.access(os.path.dirname(dest), os.W_OK):
os.remove(tmpsrc)
module.fail_json( msg="Destination dir %s not writable" % (os.path.dirname(dest)))
module.fail_json(msg="Destination dir %s not writable" % (os.path.dirname(dest)))
if checksum_src != checksum_dest:
try:
@ -274,55 +270,45 @@ def url_filename(url):
return fn
def uri(module, url, dest, user, password, body, body_format, method, headers, redirects, socket_timeout, validate_certs):
# To debug
#httplib2.debuglevel = 4
def absolute_location(url, location):
"""Attempts to create an absolute URL based on initial URL, and
next URL, specifically in the case of a ``Location`` header.
"""
if '://' in location:
return location
elif location.startswith('/'):
parts = urlparse.urlsplit(url)
base = url.replace(parts[2], '')
return '%s%s' % (base, location)
elif not location.startswith('/'):
base = os.path.dirname(url)
return '%s/%s' % (base, location)
# Handle Redirects
if redirects == "all" or redirects == "yes":
follow_redirects = True
follow_all_redirects = True
elif redirects == "none":
follow_redirects = False
follow_all_redirects = False
else:
follow_redirects = True
follow_all_redirects = False
return location
# Create a Http object and set some default options.
disable_validation = not validate_certs
h = httplib2.Http(disable_ssl_certificate_validation=disable_validation, timeout=socket_timeout)
h.follow_all_redirects = follow_all_redirects
h.follow_redirects = follow_redirects
h.forward_authorization_headers = True
# If they have a username or password verify they have both, then add them to the request
if user is not None and password is None:
module.fail_json(msg="Both a username and password need to be set.")
if password is not None and user is None:
module.fail_json(msg="Both a username and password need to be set.")
if user is not None and password is not None:
h.add_credentials(user, password)
def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
# is dest is set and is a directory, let's check if we get redirected and
# set the filename from that url
redirected = False
resp_redir = {}
redir_info = {}
r = {}
if dest is not None:
dest = os.path.expanduser(dest)
if os.path.isdir(dest):
# first check if we are redirected to a file download
h.follow_redirects=False
# Try the request
try:
resp_redir, content_redir = h.request(url, method=method, body=body, headers=headers)
_, redir_info = fetch_url(module, url, data=body,
headers=headers,
method=method, follow_redirects=None,
timeout=socket_timeout)
# if we are redirected, update the url with the location header,
# and update dest with the new url filename
except:
pass
if 'status' in resp_redir and resp_redir['status'] in ["301", "302", "303", "307"]:
url = resp_redir['location']
if redir_info['status'] in (301, 302, 303, 307):
url = redir_info['location']
redirected = True
dest = os.path.join(dest, url_filename(url))
# if destination file already exist, only download if file newer
@ -331,85 +317,54 @@ def uri(module, url, dest, user, password, body, body_format, method, headers, r
tstamp = t.strftime('%a, %d %b %Y %H:%M:%S +0000')
headers['If-Modified-Since'] = tstamp
# do safe redirects now, including 307
h.follow_redirects=follow_redirects
# Make the request, or try to :)
resp, info = fetch_url(module, url, data=body, headers=headers,
method=method, timeout=socket_timeout)
r['redirected'] = redirected or info['url'] != url
r.update(redir_info)
r.update(info)
try:
resp, content = h.request(url, method=method, body=body, headers=headers)
r['redirected'] = redirected
r.update(resp_redir)
r.update(resp)
content = resp.read()
except AttributeError:
content = ''
return r, content, dest
except httplib2.RedirectMissingLocation:
module.fail_json(msg="A 3xx redirect response code was provided but no Location: header was provided to point to the new location.")
except httplib2.RedirectLimit:
module.fail_json(msg="The maximum number of redirections was reached without coming to a final URI.")
except httplib2.ServerNotFoundError:
module.fail_json(msg="Unable to resolve the host name given.")
except httplib2.RelativeURIError:
module.fail_json(msg="A relative, as opposed to an absolute URI, was passed in.")
except httplib2.FailedToDecompressContent:
module.fail_json(msg="The headers claimed that the content of the response was compressed but the decompression algorithm applied to the content failed.")
except httplib2.UnimplementedDigestAuthOptionError:
module.fail_json(msg="The server requested a type of Digest authentication that we are unfamiliar with.")
except httplib2.UnimplementedHmacDigestAuthOptionError:
module.fail_json(msg="The server requested a type of HMACDigest authentication that we are unfamiliar with.")
except httplib2.UnimplementedHmacDigestAuthOptionError:
module.fail_json(msg="The server requested a type of HMACDigest authentication that we are unfamiliar with.")
except httplib2.CertificateHostnameMismatch:
module.fail_json(msg="The server's certificate does not match with its hostname.")
except httplib2.SSLHandshakeError:
module.fail_json(msg="Unable to validate server's certificate against available CA certs.")
except socket.error, e:
module.fail_json(msg="Socket error: %s to %s" % (e, url))
def main():
module = AnsibleModule(
argument_spec = dict(
url = dict(required=True),
argument_spec = url_argument_spec()
argument_spec.update(dict(
dest = dict(required=False, default=None, type='path'),
user = dict(required=False, default=None),
password = dict(required=False, default=None),
url_username = dict(required=False, default=None, aliases=['user']),
url_password = dict(required=False, default=None, aliases=['password']),
body = dict(required=False, default=None),
body_format = dict(required=False, default='raw', choices=['raw', 'json']),
method = dict(required=False, default='GET', choices=['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'PATCH', 'TRACE', 'CONNECT', 'REFRESH']),
return_content = dict(required=False, default='no', type='bool'),
force_basic_auth = dict(required=False, default='no', type='bool'),
follow_redirects = dict(required=False, default='safe', choices=['all', 'safe', 'none', 'yes', 'no']),
creates = dict(required=False, default=None, type='path'),
removes = dict(required=False, default=None, type='path'),
status_code = dict(required=False, default=[200], type='list'),
timeout = dict(required=False, default=30, type='int'),
validate_certs = dict(required=False, default=True, type='bool'),
),
headers = dict(required=False, type='dict')
))
module = AnsibleModule(
argument_spec=argument_spec,
check_invalid_arguments=False,
add_file_common_args=True
)
if not HAS_HTTPLIB2:
module.fail_json(msg="httplib2 >= 0.7 is not installed")
if not HAS_URLPARSE:
module.fail_json(msg="urlparse is not installed")
url = module.params['url']
user = module.params['user']
password = module.params['password']
body = module.params['body']
body_format = module.params['body_format'].lower()
method = module.params['method']
dest = module.params['dest']
return_content = module.params['return_content']
force_basic_auth = module.params['force_basic_auth']
redirects = module.params['follow_redirects']
creates = module.params['creates']
removes = module.params['removes']
status_code = [int(x) for x in list(module.params['status_code'])]
socket_timeout = module.params['timeout']
validate_certs = module.params['validate_certs']
dict_headers = {}
dict_headers = module.params['headers']
if body_format == 'json':
# Encode the body unless its a string, then assume it is preformatted JSON
@ -417,20 +372,20 @@ def main():
body = json.dumps(body)
dict_headers['Content-Type'] = 'application/json'
# Grab all the http headers. Need this hack since passing multi-values is currently a bit ugly. (e.g. headers='{"Content-Type":"application/json"}')
# Grab all the http headers. Need this hack since passing multi-values is
# currently a bit ugly. (e.g. headers='{"Content-Type":"application/json"}')
for key, value in module.params.iteritems():
if key.startswith("HEADER_"):
skey = key.replace("HEADER_", "")
dict_headers[skey] = value
if creates is not None:
# do not run the command if the line contains creates=filename
# and the filename already exists. This allows idempotence
# of uri executions.
if os.path.exists(creates):
module.exit_json(stdout="skipped, since %s exists" % creates, changed=False, stderr=False, rc=0)
module.exit_json(stdout="skipped, since %s exists" % creates,
changed=False, stderr=False, rc=0)
if removes is not None:
# do not run the command if the line contains removes=filename
@ -439,16 +394,9 @@ def main():
if not os.path.exists(removes):
module.exit_json(stdout="skipped, since %s does not exist" % removes, changed=False, stderr=False, rc=0)
# httplib2 only sends authentication after the server asks for it with a 401.
# Some 'basic auth' servies fail to send a 401 and require the authentication
# up front. This creates the Basic authentication header and sends it immediately.
if force_basic_auth:
dict_headers["Authorization"] = "Basic {0}".format(base64.b64encode("{0}:{1}".format(user, password)))
# Make the request
resp, content, dest = uri(module, url, dest, user, password, body, body_format, method, dict_headers, redirects, socket_timeout, validate_certs)
resp, content, dest = uri(module, url, dest, body, body_format, method,
dict_headers, socket_timeout)
resp['status'] = int(resp['status'])
# Write the file out if requested
@ -467,12 +415,18 @@ def main():
else:
changed = False
# Transmogrify the headers, replacing '-' with '_', since variables dont work with dashes.
# Transmogrify the headers, replacing '-' with '_', since variables dont
# work with dashes.
uresp = {}
for key, value in resp.iteritems():
ukey = key.replace("-", "_")
uresp[ukey] = value
try:
uresp['location'] = absolute_location(url, uresp['location'])
except KeyError:
pass
# Default content_encoding to try
content_encoding = 'utf-8'
if 'content_type' in uresp:
@ -491,7 +445,8 @@ def main():
u_content = unicode(content, content_encoding, errors='replace')
if resp['status'] not in status_code:
module.fail_json(msg="Status code was not " + str(status_code), content=u_content, **uresp)
uresp['msg'] = 'Status code was not %s: %s' % (status_code, uresp.get('msg', ''))
module.fail_json(content=u_content, **uresp)
elif return_content:
module.exit_json(changed=changed, content=u_content, **uresp)
else:
@ -500,5 +455,7 @@ def main():
# import module snippets
from ansible.module_utils.basic import *
from ansible.module_utils.urls import *
if __name__ == '__main__':
main()