From 2018a61489c56e44fdd551445121ac634a4b9177 Mon Sep 17 00:00:00 2001 From: Alberto Murillo Date: Mon, 22 Jan 2018 06:33:07 -0600 Subject: [PATCH] archive: Add support for xz (#34110) * Add support for xz format on archive module Fixes #34037 Fixes #34119 Signed-off-by: Alberto Murillo --- lib/ansible/modules/files/archive.py | 45 +++++++- .../targets/archive/tasks/main.yml | 104 +++++++++++++++++- 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/lib/ansible/modules/files/archive.py b/lib/ansible/modules/files/archive.py index 176ebc9c3b3..b14907aa42a 100644 --- a/lib/ansible/modules/files/archive.py +++ b/lib/ansible/modules/files/archive.py @@ -30,7 +30,8 @@ options: format: description: - The type of compression to use. - choices: [ bz2, gz, tar, zip ] + - Support for xz was added in version 2.5. + choices: [ bz2, gz, tar, xz, zip ] default: gz dest: description: @@ -49,8 +50,9 @@ options: author: - Ben Doherty (@bendoh) notes: - - requires tarfile, zipfile, gzip, and bzip2 packages on target host - - can produce I(gzip), I(bzip2) and I(zip) compressed files or archives + - requires tarfile, zipfile, gzip and bzip2 packages on target host + - requires lzma or backports.lzma if using xz format + - can produce I(gzip), I(bzip2), I(lzma) and I(zip) compressed files or archives ''' EXAMPLES = ''' @@ -133,6 +135,7 @@ import bz2 import filecmp import glob import gzip +import io import os import re import shutil @@ -142,13 +145,27 @@ from traceback import format_exc from ansible.module_utils.basic import AnsibleModule from ansible.module_utils._text import to_native +from ansible.module_utils.six import PY3 + +if PY3: + try: + import lzma + HAS_LZMA = True + except ImportError: + HAS_LZMA = False +else: + try: + from backports import lzma + HAS_LZMA = True + except ImportError: + HAS_LZMA = False def main(): module = AnsibleModule( argument_spec=dict( path=dict(type='list', required=True), - format=dict(type='str', default='gz', choices=['bz2', 'gz', 'tar', 'zip']), + format=dict(type='str', default='gz', choices=['bz2', 'gz', 'tar', 'xz', 'zip']), dest=dict(type='path'), exclude_path=dict(type='list'), remove=dict(type='bool', default=False), @@ -175,6 +192,10 @@ def main(): archive = False successes = [] + # Fail early + if not HAS_LZMA and format == 'xz': + module.fail_json(msg="lzma or backports.lzma is required when using xz format.") + for path in paths: path = os.path.expanduser(os.path.expandvars(path)) @@ -251,7 +272,7 @@ def main(): # No source files were found but the named archive exists: are we 'compress' or 'archive' now? if len(missing) == len(expanded_paths) and dest and os.path.exists(dest): # Just check the filename to know if it's an archive or simple compressed file - if re.search(r'(\.tar|\.tar\.gz|\.tgz|.tbz2|\.tar\.bz2|\.zip)$', os.path.basename(dest), re.IGNORECASE): + if re.search(r'(\.tar|\.tar\.gz|\.tgz|\.tbz2|\.tar\.bz2|\.tar\.xz|\.zip)$', os.path.basename(dest), re.IGNORECASE): state = 'archive' else: state = 'compress' @@ -287,6 +308,12 @@ def main(): elif format == 'gz' or format == 'bz2': arcfile = tarfile.open(dest, 'w|' + format) + # python3 tarfile module allows xz format but for python2 we have to create the tarfile + # in memory and then compress it with lzma. + elif format == 'xz': + arcfileIO = io.BytesIO() + arcfile = tarfile.open(fileobj=arcfileIO, mode='w') + # Or plain tar archiving elif format == 'tar': arcfile = tarfile.open(dest, 'w') @@ -342,6 +369,11 @@ def main(): arcfile.close() state = 'archive' + if format == 'xz': + with lzma.open(dest, 'wb') as f: + f.write(arcfileIO.getvalue()) + arcfileIO.close() + if errors: module.fail_json(msg='Errors when writing archive at %s: %s' % (dest, '; '.join(errors))) @@ -402,6 +434,8 @@ def main(): f_out = gzip.open(dest, 'wb') elif format == 'bz2': f_out = bz2.BZ2File(dest, 'wb') + elif format == 'xz': + f_out = lzma.LZMAFile(dest, 'wb') else: raise OSError("Invalid format") @@ -447,5 +481,6 @@ def main(): expanded_paths=expanded_paths, expanded_exclude_paths=expanded_exclude_paths) + if __name__ == '__main__': main() diff --git a/test/integration/targets/archive/tasks/main.yml b/test/integration/targets/archive/tasks/main.yml index ee42757df6c..5cd9735e73c 100644 --- a/test/integration/targets/archive/tasks/main.yml +++ b/test/integration/targets/archive/tasks/main.yml @@ -25,6 +25,44 @@ apt: name=zip state=latest when: ansible_pkg_mgr == 'apt' +- name: Install prerequisites for backports.lzma when using python2 (non OSX) + block: + - name: Set liblzma package name depending on the OS + set_fact: + liblzma_dev_package: + Debian: liblzma-dev + RedHat: xz-devel + Suse: xz-devel + - name: Ensure liblzma-dev is present to install backports-lzma + package: name={{ liblzma_dev_package[ansible_os_family] }} state=latest + when: ansible_os_family in liblzma_dev_package.keys() + when: + - ansible_python_version.split('.')[0] == '2' + - ansible_os_family != 'Darwin' + +- name: Install prerequisites for backports.lzma when using python2 (OSX) + block: + - name: Find brew binary + command: which brew + register: brew_which + - name: Get owner of brew binary + stat: path="{{ brew_which.stdout }}" + register: brew_stat + - name: "Install package" + homebrew: + name: xz + state: present + update_homebrew: no + become: yes + become_user: "{{ brew_stat.stat.pw_name }}" + when: + - ansible_python_version.split('.')[0] == '2' + - ansible_os_family == 'Darwin' + +- name: Ensure backports.lzma is present to create test archive (pip) + pip: name=backports.lzma state=latest + when: ansible_python_version.split('.')[0] == '2' + - name: prep our file copy: src={{ item }} dest={{output_dir}}/{{ item }} with_items: @@ -81,13 +119,32 @@ - name: verify that the files archived file: path={{output_dir}}/archive_01.bz2 state=file -- name: check if zip file exists +- name: check if bzip file exists assert: that: - "{{ archive_bz2_result_01.changed }}" - "{{ 'archived' in archive_bz2_result_01 }}" - "{{ archive_bz2_result_01['archived'] | length }} == 2" +- name: archive using xz + archive: + path: "{{ output_dir }}/*.txt" + dest: "{{ output_dir }}/archive_01.xz" + format: xz + register: archive_xz_result_01 + +- debug: msg="{{ archive_xz_result_01 }}" + +- name: verify that the files archived + file: path={{output_dir}}/archive_01.xz state=file + +- name: check if xz file exists + assert: + that: + - "{{ archive_xz_result_01.changed }}" + - "{{ 'archived' in archive_xz_result_01 }}" + - "{{ archive_xz_result_01['archived'] | length }} == 2" + - name: archive and set mode to 0600 archive: path: "{{ output_dir }}/*.txt" @@ -164,6 +221,30 @@ - name: remove our bz2 file: path="{{ output_dir }}/archive_02.bz2" state=absent +- name: archive and set mode to 0600 + archive: + path: "{{ output_dir }}/*.txt" + dest: "{{ output_dir }}/archive_02.xz" + format: xz + mode: "u+rwX,g-rwx,o-rwx" + register: archive_xz_result_02 + +- name: Test that the file modes were changed + stat: + path: "{{ output_dir }}/archive_02.xz" + register: archive_02_xz_stat + +- name: Test that the file modes were changed + assert: + that: + - "archive_02_xz_stat.changed == False" + - "archive_02_xz_stat.stat.mode == '0600'" + - "'archived' in archive_xz_result_02" + - "{{ archive_xz_result_02['archived']| length}} == 2" + +- name: remove our xz + file: path="{{ output_dir }}/archive_02.xz" state=absent + - name: test that gz archive that contains non-ascii filenames archive: path: "{{ output_dir }}/*.txt" @@ -206,6 +287,27 @@ - name: remove nonascii test file: path="{{ output_dir }}/test-archive-nonascii-くらとみ.bz2" state=absent +- name: test that xz archive that contains non-ascii filenames + archive: + path: "{{ output_dir }}/*.txt" + dest: "{{ output_dir }}/test-archive-nonascii-くらとみ.xz" + format: xz + register: nonascii_result_1 + +- name: Check that file is really there + stat: + path: "{{ output_dir }}/test-archive-nonascii-くらとみ.xz" + register: nonascii_stat_1 + +- name: Assert that nonascii tests succeeded + assert: + that: + - "nonascii_result_1.changed == true" + - "nonascii_stat_1.stat.exists == true" + +- name: remove nonascii test + file: path="{{ output_dir }}/test-archive-nonascii-くらとみ.xz" state=absent + - name: test that zip archive that contains non-ascii filenames archive: path: "{{ output_dir }}/*.txt"