cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Filesystem: refactor, improvements, add tests (#25519)

Browse source 
* filesystem: list used tools

* filesystem: btrfs and reiserfs don't support resizing

* filesystem: list supported filesystems

use formatting functions and 'filesystem' instead of 'file system'

* filesystem: PEP8

* filesystem: remove useless calls to module.boolean

* filesystem: fail when the requested action isn't implemented

* filesystem: resizefs: list supported FS rather than unsupported

* filesystem: refactor

* filesystem: add integration tests

* filesystem: allow to use image file with 'dev' param

* filesystem: test resizefs (ext2/3/4 filesystems only)

* filesystem: Btrfs, handle older version than v0.20-rc1

* filesystem: use loop keyword (integration tests)

* filesystem: new test, check when another filesystem already exists

* filesystem: add myself as a maintainer

* filesystem: fix tests as filters
This commit is contained in:
Pilou 2018-01-11 17:10:14 +01:00 committed by ansibot
parent 303894dd17
commit e9e316c76b
8 changed files with 410 additions and 171 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/BOTMETA.yml vendored
View file

@ -627,7 +627,7 @@ files:
$modules/system/dconf.py: azaghal
$modules/system/debconf.py: bcoca
$modules/system/facter.py: $team_ansible
$modules/system/filesystem.py: abulimov
$modules/system/filesystem.py: abulimov pilou-
$modules/system/firewalld.py: maxamillion
$modules/system/gconftool2.py: akasurde kevensen
$modules/system/getent.py: bcoca

366
lib/ansible/modules/system/filesystem.py
View file

@ -16,21 +16,22 @@ DOCUMENTATION = '''
author:
- Alexander Bulimov (@abulimov)
module: filesystem
short_description: Makes file system on block device
short_description: Makes a filesystem
description:
- This module creates file system.
- This module creates a filesystem.
version_added: "1.2"
options:
fstype:
choices: [ btrfs, ext2, ext3, ext4, ext4dev, lvm, reiserfs, xfs ]
description:
- File System type to be created.
- Filesystem type to be created.
- reiserfs support was added in 2.2.
- lvm support was added in 2.5.
- since 2.5, I(dev) can be an image file.
required: yes
dev:
description:
- Target block device.
- Target path to device or image file.
required: yes
force:
description:
@ -39,16 +40,20 @@ options:
default: 'no'
resizefs:
description:
- If C(yes), if the block device and filessytem size differ, grow the filesystem into the space.
- Note, XFS Will only grow if mounted.
- If C(yes), if the block device and filesytem size differ, grow the filesystem into the space.
- Supported for C(ext2), C(ext3), C(ext4), C(ext4dev), C(lvm) and C(xfs) filesystems.
- XFS Will only grow if mounted.
type: bool
default: 'no'
version_added: "2.0"
opts:
description:
- List of options to be passed to mkfs command.
requirements:
- Uses tools related to the I(fstype) (C(mkfs)) and C(blkid) command. When I(resizefs) is enabled, C(blockdev) command is required too.
notes:
- Uses mkfs command.
- Potential filesystem on I(dev) are checked using C(blkid), in case C(blkid) isn't able to detect an existing filesystem,
this filesystem is overwritten even if I(force) is C(no).
'''
EXAMPLES = '''
@ -64,63 +69,174 @@ EXAMPLES = '''
opts: -cc
'''
from distutils.version import LooseVersion
import os
import re
import stat
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.six import viewkeys
def _get_dev_size(dev, module):
""" Return size in bytes of device. Returns int """
blockdev_cmd = module.get_bin_path("blockdev", required=True)
rc, devsize_in_bytes, err = module.run_command("%s %s %s" % (blockdev_cmd, "--getsize64", dev))
return int(devsize_in_bytes)
class Filesystem(object):
GROW = None
MKFS = None
MKFS_FORCE_FLAGS = ''
def __init__(self, module):
self.module = module
@property
def fstype(self):
return type(self).__name__
def get_dev_size(self, dev):
""" Return size in bytes of device. Returns int """
blockdev_cmd = self.module.get_bin_path("blockdev", required=True)
_, devsize_in_bytes, _ = self.module.run_command("%s %s %s" % (blockdev_cmd, "--getsize64", dev), check_rc=True)
return int(devsize_in_bytes)
def get_fs_size(self, dev):
""" Return size in bytes of filesystem on device. Returns int """
raise NotImplementedError()
def create(self, opts, dev):
if self.module.check_mode:
return
mkfs = self.module.get_bin_path(self.MKFS, required=True)
if opts is None:
cmd = "%s %s '%s'" % (mkfs, self.MKFS_FORCE_FLAGS, dev)
else:
cmd = "%s %s %s '%s'" % (mkfs, self.MKFS_FORCE_FLAGS, opts, dev)
self.module.run_command(cmd, check_rc=True)
def grow(self, dev):
"""Get dev and fs size and compare. Returns stdout of used command."""
statinfo = os.stat(dev)
if stat.S_ISBLK(statinfo.st_mode):
devsize_in_bytes = self.get_dev_size(dev)
elif os.path.isfile(dev):
devsize_in_bytes = os.path.getsize(dev)
else:
self.module.fail_json(changed=False, msg="Target device not supported: %r." % dev)
try:
fssize_in_bytes = self.get_fs_size(dev)
except NotImplementedError:
self.module.fail_json(changed=False, msg="module does not support resizing %s filesystem yet." % self.fstype)
fs_smaller = fssize_in_bytes < devsize_in_bytes
if self.module.check_mode and fs_smaller:
self.module.exit_json(changed=True, msg="Resizing filesystem %s on device %s" % (self.fstype, dev))
elif self.module.check_mode and not fs_smaller:
self.module.exit_json(changed=False, msg="%s filesystem is using the whole device %s" % (self.fstype, dev))
elif fs_smaller:
cmd = self.module.get_bin_path(self.GROW, required=True)
_, out, _ = self.module.run_command("%s %s" % (cmd, dev), check_rc=True)
return out
else:
self.module.exit_json(changed=False, msg="%s filesystem is using the whole device %s" % (self.fstype, dev))
def _get_fs_size(fssize_cmd, dev, module):
""" Return size in bytes of filesystem on device. Returns int """
cmd = module.get_bin_path(fssize_cmd, required=True)
if 'tune2fs' == fssize_cmd:
class Ext(Filesystem):
MKFS_FORCE_FLAGS = '-F'
GROW = 'resize2fs'
def get_fs_size(self, dev):
cmd = self.module.get_bin_path('tune2fs', required=True)
# Get Block count and Block size
rc, size, err = module.run_command("%s %s %s" % (cmd, '-l', dev))
if rc == 0:
for line in size.splitlines():
if 'Block count:' in line:
block_count = int(line.split(':')[1].strip())
elif 'Block size:' in line:
block_size = int(line.split(':')[1].strip())
break
else:
module.fail_json(msg="Failed to get block count and block size of %s with %s" % (dev, cmd), rc=rc, err=err)
elif 'xfs_growfs' == fssize_cmd:
# Get Block count and Block size
rc, size, err = module.run_command([cmd, '-n', dev])
if rc == 0:
for line in size.splitlines():
col = line.split('=')
if col[0].strip() == 'data':
if col[1].strip() != 'bsize':
module.fail_json(msg='Unexpected output format from xfs_growfs (could not locate "bsize")')
if col[2].split()[1] != 'blocks':
module.fail_json(msg='Unexpected output format from xfs_growfs (could not locate "blocks")')
block_size = int(col[2].split()[0])
block_count = int(col[3].split(',')[0])
break
else:
module.fail_json(msg="Failed to get block count and block size of %s with %s" % (dev, cmd), rc=rc, err=err)
elif 'btrfs' == fssize_cmd:
# ToDo
# There is no way to get the blocksize and blockcount for btrfs filesystems
block_size = 1
block_count = 1
elif 'pvs' == fssize_cmd:
rc, size, err = module.run_command([cmd, '--noheadings', '-o', 'pv_size', '--units', 'b', dev])
if rc == 0:
block_count = int(size[:-1])
block_size = 1
else:
module.fail_json(msg="Failed to get block count and block size of %s with %s" % (dev, cmd), rc=rc, err=err)
_, size, _ = self.module.run_command([cmd, '-l', dev], check_rc=True)
for line in size.splitlines():
if 'Block count:' in line:
block_count = int(line.split(':')[1].strip())
elif 'Block size:' in line:
block_size = int(line.split(':')[1].strip())
return block_size * block_count
return block_size * block_count
class Ext2(Ext):
MKFS = 'mkfs.ext2'
class Ext3(Ext):
MKFS = 'mkfs.ext3'
class Ext4(Ext):
MKFS = 'mkfs.ext4'
class XFS(Filesystem):
MKFS = 'mkfs.xfs'
MKFS_FORCE_FLAGS = '-f'
GROW = 'xfs_growfs'
def get_fs_size(self, dev):
cmd = self.module.get_bin_path('xfs_growfs', required=True)
_, size, _ = self.module.run_command([cmd, '-n', dev], check_rc=True)
for line in size.splitlines():
col = line.split('=')
if col[0].strip() == 'data':
if col[1].strip() != 'bsize':
self.module.fail_json(msg='Unexpected output format from xfs_growfs (could not locate "bsize")')
if col[2].split()[1] != 'blocks':
self.module.fail_json(msg='Unexpected output format from xfs_growfs (could not locate "blocks")')
block_size = int(col[2].split()[0])
block_count = int(col[3].split(',')[0])
return block_size * block_count
class Reiserfs(Filesystem):
MKFS = 'mkfs.reiserfs'
MKFS_FORCE_FLAGS = '-f'
class Btrfs(Filesystem):
MKFS = 'mkfs.btrfs'
def __init__(self, module):
super(Btrfs, self).__init__(module)
_, stdout, stderr = self.module.run_command('%s --version' % self.MKFS, check_rc=True)
match = re.search(r" v([0-9.]+)", stdout)
if not match:
# v0.20-rc1 use stderr
match = re.search(r" v([0-9.]+)", stderr)
if match:
# v0.20-rc1 doesn't have --force parameter added in following version v3.12
if LooseVersion(match.group(1)) >= LooseVersion('3.12'):
self.MKFS_FORCE_FLAGS = '-f'
else:
self.MKFS_FORCE_FLAGS = ''
else:
# assume version is greater or equal to 3.12
self.MKFS_FORCE_FLAGS = '-f'
self.module.warn('Unable to identify mkfs.btrfs version (%r, %r)' % (stdout, stderr))
class LVM(Filesystem):
MKFS = 'pvcreate'
MKFS_FORCE_FLAGS = '-f'
GROW = 'pvresize'
def get_fs_size(self, dev):
cmd = self.module.get_bin_path('pvs', required=True)
_, size, _ = self.module.run_command([cmd, '--noheadings', '-o', 'pv_size', '--units', 'b', dev], check_rc=True)
block_count = int(size[:-1]) # block size is 1
return block_count
FILESYSTEMS = {
'ext2': Ext2,
'ext3': Ext3,
'ext4': Ext4,
'ext4dev': Ext4,
'reiserfs': Reiserfs,
'xfs': XFS,
'btrfs': Btrfs,
'LVM2_member': LVM,
}
def main():
@ -129,71 +245,12 @@ def main():
}
# There is no "single command" to manipulate filesystems, so we map them all out and their options
fs_cmd_map = {
'ext2': {
'mkfs': 'mkfs.ext2',
'grow': 'resize2fs',
'grow_flag': None,
'force_flag': '-F',
'fsinfo': 'tune2fs',
},
'ext3': {
'mkfs': 'mkfs.ext3',
'grow': 'resize2fs',
'grow_flag': None,
'force_flag': '-F',
'fsinfo': 'tune2fs',
},
'ext4': {
'mkfs': 'mkfs.ext4',
'grow': 'resize2fs',
'grow_flag': None,
'force_flag': '-F',
'fsinfo': 'tune2fs',
},
'reiserfs': {
'mkfs': 'mkfs.reiserfs',
'grow': 'resize_reiserfs',
'grow_flag': None,
'force_flag': '-f',
'fsinfo': 'reiserfstune',
},
'ext4dev': {
'mkfs': 'mkfs.ext4',
'grow': 'resize2fs',
'grow_flag': None,
'force_flag': '-F',
'fsinfo': 'tune2fs',
},
'xfs': {
'mkfs': 'mkfs.xfs',
'grow': 'xfs_growfs',
'grow_flag': None,
'force_flag': '-f',
'fsinfo': 'xfs_growfs',
},
'btrfs': {
'mkfs': 'mkfs.btrfs',
'grow': 'btrfs',
'grow_flag': 'filesystem resize',
'force_flag': '-f',
'fsinfo': 'btrfs',
},
'LVM2_member': {
'mkfs': 'pvcreate',
'grow': 'pvresize',
'grow_flag': None,
'force_flag': '-f',
'fsinfo': 'pvs',
}
}
module = AnsibleModule(
argument_spec=dict(
fstype=dict(type='str', required=True, aliases=['type'],
choices=fs_cmd_map.keys() + friendly_names.keys()),
dev=dict(type='str', required=True, aliases=['device']),
opts=dict(type='str'),
fstype=dict(required=True, aliases=['type'],
choices=list(FILESYSTEMS.keys()) + list(friendly_names.keys())),
dev=dict(required=True, aliases=['device']),
opts=dict(),
force=dict(type='bool', default=False),
resizefs=dict(type='bool', default=False),
),
@ -203,8 +260,8 @@ def main():
dev = module.params['dev']
fstype = module.params['fstype']
opts = module.params['opts']
force = module.boolean(module.params['force'])
resizefs = module.boolean(module.params['resizefs'])
force = module.params['force']
resizefs = module.params['resizefs']
if fstype in friendly_names:
fstype = friendly_names[fstype]
@ -212,70 +269,39 @@ def main():
changed = False
try:
_ = fs_cmd_map[fstype]
klass = FILESYSTEMS[fstype]
except KeyError:
module.exit_json(changed=False, msg="WARNING: module does not support this filesystem yet. %s" % fstype)
mkfscmd = fs_cmd_map[fstype]['mkfs']
force_flag = fs_cmd_map[fstype]['force_flag']
growcmd = fs_cmd_map[fstype]['grow']
fssize_cmd = fs_cmd_map[fstype]['fsinfo']
module.fail_json(changed=False, msg="module does not support this filesystem (%s) yet." % fstype)
if not os.path.exists(dev):
module.fail_json(msg="Device %s not found." % dev)
cmd = module.get_bin_path('blkid', required=True)
rc, raw_fs, err = module.run_command("%s -c /dev/null -o value -s TYPE %s" % (cmd, dev))
# In case blkid isn't able to identify an existing filesystem, device is considered as empty,
# then this existing filesystem would be overwritten even if force isn't enabled.
fs = raw_fs.strip()
if fs == fstype and resizefs is False and not force:
module.exit_json(changed=False)
elif fs == fstype and resizefs is True:
# Get dev and fs size and compare
devsize_in_bytes = _get_dev_size(dev, module)
fssize_in_bytes = _get_fs_size(fssize_cmd, dev, module)
if fssize_in_bytes < devsize_in_bytes:
fs_smaller = True
else:
fs_smaller = False
filesystem = klass(module)
if module.check_mode and fs_smaller:
module.exit_json(changed=True, msg="Resizing filesystem %s on device %s" % (fstype, dev))
elif module.check_mode and not fs_smaller:
module.exit_json(changed=False, msg="%s filesystem is using the whole device %s" % (fstype, dev))
elif fs_smaller:
cmd = module.get_bin_path(growcmd, required=True)
rc, out, err = module.run_command("%s %s" % (cmd, dev))
# Sadly there is no easy way to determine if this has changed. For now, just say "true" and move on.
# in the future, you would have to parse the output to determine this.
# thankfully, these are safe operations if no change is made.
if rc == 0:
module.exit_json(changed=True, msg=out)
else:
module.fail_json(msg="Resizing filesystem %s on device '%s' failed" % (fstype, dev), rc=rc, err=err)
else:
module.exit_json(changed=False, msg="%s filesystem is using the whole device %s" % (fstype, dev))
same_fs = fs and FILESYSTEMS.get(fs) == FILESYSTEMS[fstype]
if same_fs and not resizefs and not force:
module.exit_json(changed=False)
elif same_fs and resizefs:
if not filesystem.GROW:
module.fail_json(changed=False, msg="module does not support resizing %s filesystem yet." % fstype)
out = filesystem.grow(dev)
# Sadly there is no easy way to determine if this has changed. For now, just say "true" and move on.
# in the future, you would have to parse the output to determine this.
# thankfully, these are safe operations if no change is made.
module.exit_json(changed=True, msg=out)
elif fs and not force:
module.fail_json(msg="'%s' is already used as %s, use force=yes to overwrite" % (dev, fs), rc=rc, err=err)
# create fs
if module.check_mode:
changed = True
else:
mkfs = module.get_bin_path(mkfscmd, required=True)
cmd = None
if opts is None:
cmd = "%s %s '%s'" % (mkfs, force_flag, dev)
else:
cmd = "%s %s %s '%s'" % (mkfs, force_flag, opts, dev)
rc, _, err = module.run_command(cmd)
if rc == 0:
changed = True
else:
module.fail_json(msg="Creating filesystem %s on device '%s' failed" % (fstype, dev), rc=rc, err=err)
filesystem.create(opts, dev)
changed = True
module.exit_json(changed=changed)

3
test/integration/targets/filesystem/aliases Normal file
View file

@ -0,0 +1,3 @@
destructive
posix/ci/group3
skip/osx

14
test/integration/targets/filesystem/defaults/main.yml Normal file
View file

@ -0,0 +1,14 @@
tested_filesystems:
# key: fstype
# fssize: size (Mo)
# grow: True if resizefs is supported
# Other minimal sizes:
# - XFS: 20Mo
# - Btrfs: 100Mo (50Mo when "--metadata single" is used)
ext4: {fssize: 10, grow: True}
ext4dev: {fssize: 10, grow: True}
ext3: {fssize: 10, grow: True}
ext2: {fssize: 10, grow: True}
xfs: {fssize: 20, grow: False} # grow requires a mounted filesystem
btrfs: {fssize: 100, grow: False} # grow not implemented
# untested: lvm, requires a block device

88
test/integration/targets/filesystem/tasks/create_fs.yml Normal file
View file

@ -0,0 +1,88 @@
- block:
- name: 'Create a "disk" file'
command: 'dd if=/dev/zero of={{ dev }} bs=1M count={{ fssize }}'
- name: filesystem creation
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
register: fs_result
- assert:
that:
- 'fs_result is changed'
- 'fs_result is success'
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid
- name: "Check that filesystem isn't created if force isn't used"
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
register: fs2_result
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid2
- assert:
that:
- 'not (fs2_result is changed)'
- 'fs2_result is success'
- 'uuid.stdout == uuid2.stdout'
- name: Check that filesystem is recreated if force is used
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
force: yes
register: fs3_result
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid3
- assert:
that:
- 'fs3_result is changed'
- 'fs3_result is success'
- 'uuid.stdout != uuid3.stdout'
- name: increase fake device
shell: 'dd if=/dev/zero bs=1M count=20 >> {{ dev }}'
- when: 'grow|bool'
block:
- name: Expand filesystem
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
resizefs: yes
register: fs4_result
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid4
- assert:
that:
- 'fs4_result is changed'
- 'fs4_result is success'
- 'uuid3.stdout == uuid4.stdout' # unchanged
- name: Try to expand filesystem again
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
resizefs: yes
register: fs5_result
- assert:
that:
- 'not (fs5_result is changed)'
- 'fs5_result is successful'
- import_tasks: overwrite_another_fs.yml
always:
- file:
name: '{{ dev }}'
state: absent

19
test/integration/targets/filesystem/tasks/main.yml Normal file
View file

@ -0,0 +1,19 @@
- debug:
msg: '{{ role_name }}'
- debug:
msg: '{{ role_path|basename }}'
- import_tasks: setup.yml
- include_tasks: create_fs.yml
vars:
dev: '{{ ansible_user_dir }}/ansible_testing/img'
fstype: '{{ item.key }}'
fssize: '{{ item.value.fssize }}'
grow: '{{ item.value.grow }}'
when:
- 'not (item.key == "btrfs" and ansible_system == "FreeBSD")'
# On Ubuntu trusty, blkid is unable to identify filesystem smaller than 256Mo, see:
# https://www.kernel.org/pub/linux/utils/util-linux/v2.21/v2.21-ChangeLog
# https://anonscm.debian.org/cgit/collab-maint/pkg-util-linux.git/commit/?id=04f7020eadf31efc731558df92daa0a1c336c46c
- 'not (item.key == "btrfs" and (ansible_distribution == "Ubuntu" and ansible_distribution_release == "trusty"))'
loop: "{{ lookup('dict', tested_filesystems) }}"

44
test/integration/targets/filesystem/tasks/overwrite_another_fs.yml Normal file
View file

@ -0,0 +1,44 @@
- name: 'Recreate "disk" file'
command: 'dd if=/dev/zero of={{ dev }} bs=1M count={{ fssize }}'
- name: 'Create a vfat filesystem'
command: 'mkfs.vfat {{ dev }}'
when: ansible_system != 'FreeBSD'
- name: 'Create a vfat filesystem'
command: 'newfs_msdos -F12 {{ dev }}'
when: ansible_system == 'FreeBSD'
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid
- name: "Check that an existing filesystem (not handled by this module) isn't overwritten when force isn't used"
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
register: fs_result
ignore_errors: True
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid2
- assert:
that:
- 'fs_result is failed'
- 'uuid.stdout == uuid2.stdout'
- name: "Check that an existing filesystem (not handled by this module) is overwritten when force is used"
filesystem:
dev: '{{ dev }}'
fstype: '{{ fstype }}'
force: yes
register: fs_result2
- command: 'blkid -c /dev/null -o value -s UUID {{ dev }}'
register: uuid3
- assert:
that:
- 'fs_result2 is successful'
- 'fs_result2 is changed'
- 'uuid2.stdout != uuid3.stdout'

45
test/integration/targets/filesystem/tasks/setup.yml Normal file
View file

@ -0,0 +1,45 @@
- name: install filesystem tools
package:
name: '{{ item }}'
state: present
when: ansible_system == 'Linux' or item != 'dosfstools'
with_items:
- e2fsprogs
- xfsprogs
- dosfstools
- block:
- name: install btrfs progs
package:
name: btrfs-progs
state: present
when: ansible_os_family != 'Suse' and not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('16.04', '<='))
- name: install btrfs progs (Ubuntu <= 16.04)
package:
name: btrfs-tools
state: present
when: ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('16.04', '<=')
- name: install btrfs progs (OpenSuse)
package:
name: '{{ item }}'
state: present
when: ansible_os_family == 'Suse'
with_items:
- python-xml
- btrfsprogs
when: ansible_system == 'Linux'
- command: mke2fs -V
register: mke2fs
- set_fact:
# mke2fs 1.43.6 (29-Aug-2017)
e2fsprogs_version: '{{ mke2fs.stderr_lines[0] | regex_search("[0-9]{1,2}\.[0-9]{1,2}(\.[0-9]{1,2})?") }}'
- set_fact:
# http://e2fsprogs.sourceforge.net/e2fsprogs-release.html#1.43
# Mke2fs no longer complains if the user tries to create a file system
# using the entire block device.
force_creation: "{{ e2fsprogs_version is version('1.43', '<') }}"