cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

introduce module_utils.urls.fetch_file as a wrapper to download and save files (#19172)

Browse source 
* module_utils.urls: add fetch_file function

* apt: use fetch_file instead of own download()

* unarchive: use fetch_file instead of own codecopy

* apt: add test for deb=http://…

* unarchive: add test for a remote file download and unarchive

* yum: replace fetch_rpm_from_url by fetch_file

* use NamedTemporaryFile

* don't add a dot to fileext, it's already there
This commit is contained in:
Evgeni Golov 2018-10-08 14:41:57 +02:00 committed by Martin Krizek
parent c8ed5c29e9
commit 7c66c90afc
6 changed files with 86 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
lib/ansible/module_utils/urls.py
View file

@ -1324,3 +1324,40 @@ def fetch_url(module, url, data=None, headers=None, method=None,
tempfile.tempdir = old_tempdir
return r, info
def fetch_file(module, url, data=None, headers=None, method=None,
use_proxy=True, force=False, last_mod_time=None, timeout=10):
'''Download and save a file via HTTP(S) or FTP (needs the module as parameter).
This is basically a wrapper around fetch_url().
:arg module: The AnsibleModule (used to get username, password etc. (s.b.).
:arg url: The url to use.
:kwarg data: The data to be sent (in case of POST/PUT).
:kwarg headers: A dict with the request headers.
:kwarg method: "POST", "PUT", etc.
:kwarg boolean use_proxy: Default: True
:kwarg boolean force: If True: Do not get a cached copy (Default: False)
:kwarg last_mod_time: Default: None
:kwarg int timeout: Default: 10
:returns: A string, the path to the downloaded file.
'''
# download file
bufsize = 65536
file_name, file_ext = os.path.splitext(str(url.rsplit('/', 1)[1]))
fetch_temp_file = tempfile.NamedTemporaryFile(dir=module.tmpdir, prefix=file_name, suffix=file_ext, delete=False)
module.add_cleanup_file(fetch_temp_file.name)
try:
rsp, info = fetch_url(module, url, data, headers, method, use_proxy, force, last_mod_time, timeout)
if not rsp:
module.fail_json(msg="Failure downloading %s, %s" % (url, info['msg']))
data = rsp.read(bufsize)
while data:
fetch_temp_file.write(data)
data = rsp.read(bufsize)
fetch_temp_file.close()
except Exception as e:
module.fail_json(msg="Failure downloading %s, %s" % (url, to_native(e)))
return fetch_temp_file.name

28
lib/ansible/modules/files/unarchive.py
View file

@ -144,7 +144,7 @@ import traceback
from zipfile import ZipFile, BadZipfile
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.urls import fetch_url
from ansible.module_utils.urls import fetch_file
from ansible.module_utils._text import to_bytes, to_native, to_text
try: # python 3.3+
@ -162,9 +162,6 @@ MOD_TIME_DIFF_RE = re.compile(r': Mod time differs$')
EMPTY_FILE_RE = re.compile(r': : Warning: Cannot stat: No such file or directory$')
MISSING_FILE_RE = re.compile(r': Warning: Cannot stat: No such file or directory$')
ZIP_FILE_MODE_RE = re.compile(r'([r-][w-][SsTtx-]){3}')
# When downloading an archive, how much of the archive to download before
# saving to a tempfile (64k)
BUFSIZE = 65536
def crc32(path):
@ -821,28 +818,7 @@ def main():
module.fail_json(msg="Source '%s' failed to transfer" % src)
# If remote_src=true, and src= contains ://, try and download the file to a temp directory.
elif '://' in src:
new_src = os.path.join(module.tmpdir, to_native(src.rsplit('/', 1)[1], errors='surrogate_or_strict'))
try:
rsp, info = fetch_url(module, src)
# If download fails, raise a proper exception
if rsp is None:
raise Exception(info['msg'])
# open in binary mode for python3
f = open(new_src, 'wb')
# Read 1kb at a time to save on ram
while True:
data = rsp.read(BUFSIZE)
data = to_bytes(data, errors='surrogate_or_strict')
if len(data) < 1:
break # End of file, break while loop
f.write(data)
f.close()
src = new_src
except Exception as e:
module.fail_json(msg="Failure downloading %s, %s" % (src, to_native(e)))
src = fetch_file(module, src)
else:
module.fail_json(msg="Source '%s' does not exist" % src)
if not os.access(src, os.R_OK):

34
lib/ansible/modules/packaging/os/apt.py
View file

@ -266,7 +266,7 @@ import time
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.urls import fetch_url
from ansible.module_utils.urls import fetch_file
# APT related constants
APT_ENV_VARS = dict(
@ -846,36 +846,6 @@ def upgrade(m, mode="yes", force=False, default_release=None,
m.exit_json(changed=True, msg=out, stdout=out, stderr=err, diff=diff)
def download(module, deb):
package = os.path.join(module.tmpdir, to_native(deb.rsplit('/', 1)[1], errors='surrogate_or_strict'))
# When downloading a deb, how much of the deb to download before
# saving to a tempfile (64k)
BUFSIZE = 65536
try:
rsp, info = fetch_url(module, deb, method='GET')
if info['status'] != 200:
module.fail_json(msg="Failed to download %s, %s" % (deb,
info['msg']))
# Ensure file is open in binary mode for Python 3
f = open(package, 'wb')
# Read 1kb at a time to save on ram
while True:
data = rsp.read(BUFSIZE)
data = to_bytes(data, errors='surrogate_or_strict')
if len(data) < 1:
break # End of file, break while loop
f.write(data)
f.close()
deb = package
except Exception as e:
module.fail_json(msg="Failure downloading %s, %s" % (deb, to_native(e)))
return deb
def get_cache_mtime():
"""Return mtime of a valid apt cache file.
Stat the apt cache file and if no cache file is found return 0
@ -1053,7 +1023,7 @@ def main():
if p['state'] != 'present':
module.fail_json(msg="deb only supports state=present")
if '://' in p['deb']:
p['deb'] = download(module, p['deb'])
p['deb'] = fetch_file(module, p['deb'])
install_deb(module, p['deb'], cache,
install_recommends=install_recommends,
allow_unauthenticated=allow_unauthenticated,

30
lib/ansible/modules/packaging/os/yum.py
View file

@ -352,13 +352,11 @@ except ImportError:
transaction_helpers = False
from contextlib import contextmanager
from ansible.module_utils.urls import fetch_file
def_qf = "%{epoch}:%{name}-%{version}-%{release}.%{arch}"
rpmbin = None
# 64k. Number of bytes to read at a time when manually downloading pkgs via a url
BUFSIZE = 65536
class YumModule(YumDnf):
"""
@ -384,28 +382,6 @@ class YumModule(YumDnf):
self.pkg_mgr_name = "yum"
self.lockfile = '/var/run/yum.pid'
def fetch_rpm_from_url(self, spec):
# FIXME: Remove this once this PR is merged:
# https://github.com/ansible/ansible/pull/19172
# download package so that we can query it
package_name, dummy = os.path.splitext(str(spec.rsplit('/', 1)[1]))
package_file = tempfile.NamedTemporaryFile(dir=self.module.tmpdir, prefix=package_name, suffix='.rpm', delete=False)
self.module.add_cleanup_file(package_file.name)
try:
rsp, info = fetch_url(self.module, spec)
if not rsp:
self.module.fail_json(msg="Failure downloading %s, %s" % (spec, info['msg']))
data = rsp.read(BUFSIZE)
while data:
package_file.write(data)
data = rsp.read(BUFSIZE)
package_file.close()
except Exception as e:
self.module.fail_json(msg="Failure downloading %s, %s" % (spec, to_native(e)))
return package_file.name
def yum_base(self):
my = yum.YumBase()
my.preconf.debuglevel = 0
@ -884,7 +860,7 @@ class YumModule(YumDnf):
if '://' in spec:
with self.set_env_proxy():
package = self.fetch_rpm_from_url(spec)
package = fetch_file(self.module, spec)
else:
package = spec
@ -1205,7 +1181,7 @@ class YumModule(YumDnf):
elif '://' in spec:
# download package so that we can check if it's already installed
with self.set_env_proxy():
package = self.fetch_rpm_from_url(spec)
package = fetch_file(self.module, spec)
envra = self.local_envra(package)
if envra is None:

22
test/integration/targets/apt/tasks/apt.yml
View file

@ -8,6 +8,16 @@
python_apt: python3-apt
when: ansible_python_version is version('3', '>=')
- name: use Debian mirror
set_fact:
distro_mirror: http://ftp.debian.org/debian
when: ansible_distribution == 'Debian'
- name: use Ubuntu mirror
set_fact:
distro_mirror: http://archive.ubuntu.com/ubuntu
when: ansible_distribution == 'Ubuntu'
# UNINSTALL 'python-apt'
# The `apt` module has the smarts to auto-install `python-apt`. To test, we
# will first uninstall `python-apt`.
@ -132,6 +142,18 @@
- name: uninstall hello with apt
apt: pkg=hello state=absent purge=yes
- name: install deb file from URL
apt: deb="{{ distro_mirror }}/pool/main/h/hello/hello_{{ hello_version.stdout }}_{{ hello_architecture.stdout }}.deb"
register: apt_url
- name: verify installation of hello
assert:
that:
- "apt_url.changed"
- name: uninstall hello with apt
apt: pkg=hello state=absent purge=yes
- name: force install of deb
apt: deb="/var/cache/apt/archives/hello_{{ hello_version.stdout }}_{{ hello_architecture.stdout }}.deb" force=true
register: dpkg_force

20
test/integration/targets/unarchive/tasks/main.yml
View file

@ -590,3 +590,23 @@
- name: remove our tar.gz unarchive destination
file: path={{ output_dir }}/test-unarchive-tar-gz state=absent
# Test downloading a file before unarchiving it
- name: create our unarchive destination
file: path={{output_dir}}/test-unarchive-tar-gz state=directory
- name: unarchive a tar from an URL
unarchive:
src: "https://releases.ansible.com/ansible/ansible-latest.tar.gz"
dest: "{{ output_dir }}/test-unarchive-tar-gz"
mode: "0700"
remote_src: yes
register: unarchive13
- name: Test that unarchive succeeded
assert:
that:
- "unarchive13.changed == true"
- name: remove our tar.gz unarchive destination
file: path={{ output_dir }}/test-unarchive-tar-gz state=absent