cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Tweaking daisychain internals to allow get_url to modify the path destination when downloading to a directory.

Browse source 
Minor module refactoring.
This commit is contained in:
Michael DeHaan 2012-07-22 11:08:16 -04:00
parent d0f20668a2
commit 2437ee5236
1 changed files with 71 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

185
get_url
View file

@ -17,149 +17,95 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# #
# Synopsis: # see examples/playbooks/get_url.yml
# ansible -m get_url -a "url=http://some.place/some.file dest=/tmp/file"
#
# Arguments:
# url= (mandatory, no default)
# dest= (mandatory, no default)
# if dest= is a file, url is copied to that file
# if dest= is a directory, determine name from url= and store it in dest/
# mode, owner, group, ... from the "file" module are also supported
#
# Playbook:
# The dest=<directory> feature lets you do this in a Playbook:
#
# - name: Grab a bunch of jQuery stuff
# action: get_url url=http://code.jquery.com/$item dest=${jquery_directory} mode=0444
# with_items:
# - jquery.min.js
# - mobile/latest/jquery.mobile.min.js
# - ui/jquery-ui-git.css
#
# TODO:
# timeout=
# Support gzip compression?
# http://www.diveintopython.net/http_web_services/gzip_compression.html
import sys
import os import os
import shlex
import shutil import shutil
import syslog import syslog
import datetime import datetime
import tempfile import tempfile
try:
from hashlib import md5 as _md5
except ImportError:
from md5 import md5 as _md5
HAS_URLLIB2=True HAS_URLLIB2=True
try: try:
import urllib2 import urllib2
except ImportError: except ImportError:
HAS_URLLIB2=False HAS_URLLIB2=False
HAS_URLPARSE=True HAS_URLPARSE=True
try: try:
import urlparse import urlparse
import socket import socket
except ImportError: except ImportError:
HAS_URLPARSE=False HAS_URLPARSE=False
# ==============================================================
# support
def md5(filename):
''' Return MD5 hex digest of local file, or None if file is not present. '''
if not os.path.exists(filename):
return None
digest = _md5()
blocksize = 64 * 1024
infile = open(filename, 'rb')
block = infile.read(blocksize)
while block:
digest.update(block)
block = infile.read(blocksize)
infile.close()
return digest.hexdigest()
# ============================================================== # ==============================================================
# url handling # url handling
def url_filename(url): def url_filename(url):
return os.path.basename(urlparse.urlsplit(url)[2]) fn = os.path.basename(urlparse.urlsplit(url)[2])
if fn == '':
return 'index.html'
return fn
def url_do_get(url, dest): def url_do_get(module, url, dest):
"""Get url and return request and info """
Get url and return request and info
Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp Credits: http://stackoverflow.com/questions/7006574/how-to-download-file-from-ftp
""" """
USERAGENT = 'ansible-httpget' USERAGENT = 'ansible-httpget'
info = {} info = dict(url=url)
info['url'] = url
r = None r = None
actualdest = None
if dest:
if os.path.isdir(dest): if os.path.isdir(dest):
destpath = "%s/%s" % (dest, url_filename(url)) urlfilename = url_filename(url)
actualdest = "%s/%s" % (dest, url_filename(url))
module.params['path'] = actualdest
else: else:
destpath = dest actualdest = dest
else: info['daisychain_args'] = module.params
destpath = url_filename(url) info['actualdest'] = actualdest
info['destpath'] = destpath
request = urllib2.Request(url) request = urllib2.Request(url)
request.add_header('User-agent', USERAGENT) request.add_header('User-agent', USERAGENT)
if os.path.exists(destpath): if os.path.exists(actualdest):
t = datetime.datetime.utcfromtimestamp(os.path.getmtime(destpath)) t = datetime.datetime.utcfromtimestamp(os.path.getmtime(actualdest))
tstamp = t.strftime('%a, %d %b %Y %H:%M:%S +0000') tstamp = t.strftime('%a, %d %b %Y %H:%M:%S +0000')
request.add_header('If-Modified-Since', tstamp) request.add_header('If-Modified-Since', tstamp)
try: try:
r = urllib2.urlopen(request) r = urllib2.urlopen(request)
info.update(r.info())
dinfo = dict(r.info()) info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
for x in dinfo:
info[x] = dinfo[x]
info['msg'] = "OK %s octets" % r.headers.get('Content-Length', 'unknown')
info['status'] = 200
except urllib2.HTTPError as e: except urllib2.HTTPError as e:
# Must not fail_json() here so caller can handle HTTP 304 unmodified # Must not fail_json() here so caller can handle HTTP 304 unmodified
info['msg'] = "%s" % e info.update(dict(msg=str(e), status=e.code))
info['status'] = e.code
return r, info return r, info
except urllib2.URLError as e: except urllib2.URLError as e:
if 'code' in e: code = getattr(e, 'code', -1)
co = e.code module.fail_json(msg="Request failed: %s" % str(e), status_code=code)
else:
co = -1
resp = "%s" % e
module.fail_json(msg="Request failed", status_code=co, response=resp)
return r, info return r, info
def url_get(url, dest): def url_get(module, url, dest):
"""Get url and store at dest. If dest is a directory, determine filename """
from url, otherwise dest is a file Download url and store at dest.
Return info about the request. If dest is a directory, determine filename from url.
Return (tempfile, info about the request)
""" """
req, info = url_do_get(url, dest) req, info = url_do_get(module, url, dest)
# TODO: should really handle 304, but how? src file could exist (and be
# newer) but be empty ...
# TODO: should really handle 304, but how? src file could exist (and be newer) but empty
if info['status'] == 304: if info['status'] == 304:
module.exit_json(url=url, dest=info.get('destpath', dest), changed=False, msg=info.get('msg', '')) module.exit_json(url=url, dest=info.get('actualdest', dest), changed=False, msg=info.get('msg', ''))
# We have the data. Create a temporary file and copy content into that # create a temporary file and copy content to do md5-based replacement
# to do the MD5-thing if info['status'] != 200:
module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg'], url=url)
if info['status'] == 200: actualdest = info['actualdest']
destpath = info['destpath']
fd, tempname = tempfile.mkstemp() fd, tempname = tempfile.mkstemp()
f = os.fdopen(fd, 'wb') f = os.fdopen(fd, 'wb')
@ -170,16 +116,19 @@ def url_get(url, dest):
module.fail_json(msg="failed to create temporary content file: %s" % str(err)) module.fail_json(msg="failed to create temporary content file: %s" % str(err))
f.close() f.close()
req.close() req.close()
return tempname, info return tempname, info
else:
module.fail_json(msg="Request failed", status_code=info['status'], response=info['msg'], url=url)
# ============================================================== # ==============================================================
# main # main
def main(): def main():
global module
# does this really happen on non-ancient python?
if not HAS_URLLIB2:
module.fail_json(msg="urllib2 is not installed")
if not HAS_URLPARSE:
module.fail_json(msg="urlparse is not installed")
module = AnsibleModule( module = AnsibleModule(
argument_spec = dict( argument_spec = dict(
url = dict(required=True), url = dict(required=True),
@ -187,31 +136,14 @@ def main():
) )
) )
url = module.params.get('url', '') url = module.params['url']
dest = module.params.get('dest', '') dest = os.path.expanduser(module.params['dest'])
if url == "":
module.fail_json(msg="url= URL missing")
if dest == "":
module.fail_json(msg="dest= missing")
dest = os.path.expanduser(dest)
if not HAS_URLLIB2:
module.fail_json(msg="urllib2 is not installed")
if not HAS_URLPARSE:
module.fail_json(msg="urlparse is not installed")
# Here we go... if this succeeds, tmpsrc is the name of a temporary file
# containing slurped content. If it fails, we've already raised an error
# to Ansible
tmpsrc, info = url_get(url, dest)
# download to tmpsrc
tmpsrc, info = url_get(module, url, dest)
md5sum_src = None md5sum_src = None
md5sum_dest = None
dest = info.get('destpath', None) dest = info['actualdest']
# raise an error if there is no tmpsrc file # raise an error if there is no tmpsrc file
if not os.path.exists(tmpsrc): if not os.path.exists(tmpsrc):
@ -220,10 +152,8 @@ def main():
if not os.access(tmpsrc, os.R_OK): if not os.access(tmpsrc, os.R_OK):
os.remove(tmpsrc) os.remove(tmpsrc)
module.fail_json( msg="Source %s not readable" % (tmpsrc)) module.fail_json( msg="Source %s not readable" % (tmpsrc))
md5sum_src = md5(tmpsrc) md5sum_src = module.md5(tmpsrc)
md5sum_dest = None
# check if there is no dest file # check if there is no dest file
if os.path.exists(dest): if os.path.exists(dest):
# raise an error if copy has no permission on dest # raise an error if copy has no permission on dest
@ -233,14 +163,13 @@ def main():
if not os.access(dest, os.R_OK): if not os.access(dest, os.R_OK):
os.remove(tmpsrc) os.remove(tmpsrc)
module.fail_json( msg="Destination %s not readable" % (dest)) module.fail_json( msg="Destination %s not readable" % (dest))
md5sum_dest = md5(dest) md5sum_dest = module.md5(dest)
else: else:
if not os.access(os.path.dirname(dest), os.W_OK): if not os.access(os.path.dirname(dest), os.W_OK):
os.remove(tmpsrc) os.remove(tmpsrc)
module.fail_json( msg="Destination %s not writable" % (os.path.dirname(dest))) module.fail_json( msg="Destination %s not writable" % (os.path.dirname(dest)))
if md5sum_src != md5sum_dest: if md5sum_src != md5sum_dest:
# was os.system("cp %s %s" % (src, dest))
try: try:
shutil.copyfile(tmpsrc, dest) shutil.copyfile(tmpsrc, dest)
except Exception, err: except Exception, err:
@ -250,12 +179,12 @@ def main():
else: else:
changed = False changed = False
# Mission complete
os.remove(tmpsrc) os.remove(tmpsrc)
module.exit_json(url=url, dest=dest, src=tmpsrc,
md5sum=md5sum_src, changed=changed, msg=info.get('msg', ''), # Mission complete
daisychain="file") module.exit_json(url=url, dest=dest, src=tmpsrc, md5sum=md5sum_src,
changed=changed, msg=info.get('msg',''),
daisychain="file", daisychain_args=info.get('daisychain_args',''))
# this is magic, see lib/ansible/module_common.py # this is magic, see lib/ansible/module_common.py
#<<INCLUDE_ANSIBLE_MODULE_COMMON>> #<<INCLUDE_ANSIBLE_MODULE_COMMON>>