From 45b2f4b47916be4f70ed45146d1d80892c9bea4a Mon Sep 17 00:00:00 2001 From: Vlad Glagolev Date: Sat, 10 Dec 2016 16:38:15 -0500 Subject: [PATCH] Source Mage GNU/Linux package manager support (#19072) * added alpha version of the 'sorcery' module * fully conforming YAML * use bundled check for executables * - codex_list(): use commands instead of checksums to get sorcery version and verify codex equality - renamed: - manage_depends() -> match_depends() - tocast -> cast_queue, todispel -> dispel_queue, needs_recast -> depends_ok - SORCERY_LOG -> SORCERY_LOG_DIR, SORCERY_STATE -> SORCERY_STATE_DIR - removed: - SORCERY_VERSION_FILE - CODEX - added commentary to match_depends() and manage_spells() - fixed bug about dropped dependency line for previously existed dependency - fixed bug about not fixing depends for the 'latest' state - simplified several code constructions * cleaned up some docs * do not use separate message for Codex update, rely on the 'changed' status instead * use built-in list conversion (_check_type_list()) for spells * corrected spell name extraction from list in match_depends() * avoid non-matching dependencies line duplication in depends file * added more complex playbook example * tiny stylistic fix for docs * replaced ternary construction with a regular statement * replaced yet another ternary construction with a regular statement * enable Python 2.4 compatibility by splitting try-finally block * enable Python 2.4 compatibility by replacing 'with' statement with try-except+try-finally blocks * unify spells' assign * replaced one regex with startswith() * go Ansible 2.1 * added dummy RETURN template * go Ansible 2.2 * better clarify permissions' requirements * - updated copyright years - fixed rebuild command bug - re-used run_command_environ_update dict for env var management * handle Python 3.5 * Revert "handle Python 3.5" This reverts commit 33a5a0eb64c1193318298e111f063cdd5f93b73a. * handle Python 3.5 (2nd try) * go Ansible 2.3 * clarity++ --- lib/ansible/modules/packaging/os/sorcery.py | 660 ++++++++++++++++++++ 1 file changed, 660 insertions(+) create mode 100644 lib/ansible/modules/packaging/os/sorcery.py diff --git a/lib/ansible/modules/packaging/os/sorcery.py b/lib/ansible/modules/packaging/os/sorcery.py new file mode 100644 index 00000000000..d60e9a0ec95 --- /dev/null +++ b/lib/ansible/modules/packaging/os/sorcery.py @@ -0,0 +1,660 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2015-2016, Vlad Glagolev +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + + +DOCUMENTATION = ''' +--- +module: sorcery +short_description: Package manager for Source Mage GNU/Linux +description: + - Manages "spells" on Source Mage GNU/Linux using I(sorcery) toolchain +author: "Vlad Glagolev (@vaygr)" +version_added: "2.3" +notes: + - When all three components are selected, the update goes by the sequence -- + Sorcery -> Grimoire(s) -> Spell(s); you cannot override it. + - grimoire handling (i.e. add/remove, including SCM/rsync versions) is not + yet supported. +requirements: + - bash +options: + name: + description: + - Name of the spell + - multiple names can be given, separated by commas + - special value '*' in conjunction with states C(latest) or + C(rebuild) will update or rebuild the whole system respectively + required: false + aliases: ["spell"] + default: null + + state: + description: + - Whether to cast, dispel or rebuild a package + - state C(cast) is an equivalent of C(present), not C(latest) + - state C(latest) always triggers C(update_cache=yes) + - state C(rebuild) implies cast of all specified spells, not only + those existed before + required: false + choices: ["present", "latest", "absent", "cast", "dispelled", "rebuild"] + default: "present" + + depends: + description: + - Comma-separated list of _optional_ dependencies to build a spell + (or make sure it is built) with; use +/- in front of dependency + to turn it on/off ('+' is optional though) + - this option is ignored if C(name) parameter is equal to '*' or + contains more than one spell + - providers must be supplied in the form recognized by Sorcery, e.g. + 'openssl(SSL)' + required: false + default: null + + update: + description: + - Whether or not to update sorcery scripts at the very first stage + required: false + choices: ["yes", "no"] + default: "no" + + update_cache: + description: + - Whether or not to update grimoire collection before casting spells + required: false + aliases: ["update_codex"] + choices: ["yes", "no"] + default: "no" + + cache_valid_time: + description: + - Time in seconds to invalidate grimoire collection on update + - especially useful for SCM and rsync grimoires + - makes sense only in pair with C(update_cache) + required: false + default: null +''' + + +EXAMPLES = ''' +# Make sure spell 'foo' is installed +- sorcery: + spell: foo + state: present + +# Make sure spells 'foo', 'bar' and 'baz' are removed +- sorcery: + spell: foo,bar,baz + state: absent + +# Make sure spell 'foo' with dependencies 'bar' and 'baz' is installed +- sorcery: + spell: foo + depends: bar,baz + state: present + +# Make sure spell 'foo' with 'bar' and without 'baz' dependencies is installed +- sorcery: + spell: foo + depends: +bar,-baz + state: present + +# Make sure spell 'foo' with libressl (providing SSL) dependency is installed +- sorcery: + spell: foo + depends: libressl(SSL) + state: present + +# Playbook: make sure spells with/without required dependencies (if any) are installed +- sorcery: + name: {{ item.spell }} + depends: {{ item.depends | default(None) }} + state: present + with_items: + - { spell: 'vifm', depends: '+file,-gtk+2' } + - { spell: 'fwknop', depends: 'gpgme' } + - { spell: 'pv,tnftp,tor' } + +# Install the latest version of spell 'foo' using regular glossary +- sorcery: + name: foo + state: latest + +# Rebuild spell 'foo' +- sorcery: + spell: foo + state: rebuild + +# Rebuild the whole system, but update Sorcery and Codex first +- sorcery: + spell: '*' + state: rebuild + update: yes + update_cache: yes + +# Refresh the grimoire collection if it's 1 day old using native sorcerous alias +- sorcery: + update_codex: yes + cache_valid_time: 86400 + +# Update only Sorcery itself +- sorcery: + update: yes +''' + + +RETURN = ''' +''' + + +import datetime +import fileinput +import os +import re +import shutil +import sys + + +# auto-filled at module init +SORCERY = { + 'sorcery': None, + 'scribe': None, + 'cast': None, + 'dispel': None, + 'gaze': None +} + +SORCERY_LOG_DIR = "/var/log/sorcery" +SORCERY_STATE_DIR = "/var/state/sorcery" + + +def get_sorcery_ver(module): + """ Get Sorcery version. """ + + cmd_sorcery = "%s --version" % SORCERY['sorcery'] + + rc, stdout, stderr = module.run_command(cmd_sorcery) + + if rc != 0 or not stdout: + module.fail_json(msg="unable to get Sorcery version") + + return stdout.strip() + + +def codex_fresh(codex, module): + """ Check if grimoire collection is fresh enough. """ + + if not module.params['cache_valid_time']: + return False + + timedelta = datetime.timedelta(seconds=module.params['cache_valid_time']) + + for grimoire in codex: + lastupdate_path = os.path.join(SORCERY_STATE_DIR, + grimoire + ".lastupdate") + + try: + mtime = os.stat(lastupdate_path).st_mtime + except: + return False + + lastupdate_ts = datetime.datetime.fromtimestamp(mtime) + + # if any grimoire is not fresh, we invalidate the Codex + if lastupdate_ts + timedelta < datetime.datetime.now(): + return False + + return True + + +def codex_list(module): + """ List valid grimoire collection. """ + + codex = {} + + cmd_scribe = "%s index" % SORCERY['scribe'] + + rc, stdout, stderr = module.run_command(cmd_scribe) + + if rc != 0: + module.fail_json("unable to list grimoire collection, fix your Codex") + + rex = re.compile("^\s*\[\d+\] : (?P[\w\-\+\.]+) : [\w\-\+\./]+(?: : (?P[\w\-\+\.]+))?\s*$") + + # drop 4-line header and empty trailing line + for line in stdout.splitlines()[4:-1]: + match = rex.match(line) + + if match: + codex[match.group('grim')] = match.group('ver') + + if not codex: + module.fail_json(msg="no grimoires to operate on; add at least one") + + return codex + + +def update_sorcery(module): + """ Update sorcery scripts. + + This runs 'sorcery update' ('sorcery -u'). Check mode always returns a + positive change value. + + """ + + changed = False + + if module.check_mode: + if not module.params['name'] and not module.params['update_cache']: + module.exit_json(changed=True, msg="would have updated Sorcery") + else: + sorcery_ver = get_sorcery_ver(module) + + cmd_sorcery = "%s update" % SORCERY['sorcery'] + + rc, stdout, stderr = module.run_command(cmd_sorcery) + + if rc != 0: + module.fail_json(msg="unable to update Sorcery: " + stdout) + + if sorcery_ver != get_sorcery_ver(module): + changed = True + + if not module.params['name'] and not module.params['update_cache']: + module.exit_json(changed=changed, + msg="successfully updated Sorcery") + + +def update_codex(module): + """ Update grimoire collections. + + This runs 'scribe update'. Check mode always returns a positive change + value when 'cache_valid_time' is used. + + """ + + params = module.params + + changed = False + + codex = codex_list(module) + fresh = codex_fresh(codex, module) + + if module.check_mode: + if not params['name']: + if not fresh: + changed = True + + module.exit_json(changed=changed, msg="would have updated Codex") + elif not fresh or params['name'] and params['state'] == 'latest': + # SILENT is required as a workaround for query() in libgpg + module.run_command_environ_update.update(dict(SILENT=1)) + + cmd_scribe = "%s update" % SORCERY['scribe'] + + rc, stdout, stderr = module.run_command(cmd_scribe) + + if rc != 0: + module.fail_json(msg="unable to update Codex: " + stdout) + + if codex != codex_list(module): + changed = True + + if not params['name']: + module.exit_json(changed=changed, + msg="successfully updated Codex") + + +def match_depends(module): + """ Check for matching dependencies. + + This inspects spell's dependencies with the desired states and returns + 'False' if a recast is needed to match them. It also adds required lines + to the system-wide depends file for proper recast procedure. + + """ + + params = module.params + spells = params['name'] + + depends = {} + + depends_ok = True + + if len(spells) > 1 or not params['depends']: + return depends_ok + + spell = spells[0] + + if module.check_mode: + sorcery_depends_orig = os.path.join(SORCERY_STATE_DIR, "depends") + sorcery_depends = os.path.join(SORCERY_STATE_DIR, "depends.check") + + try: + shutil.copy2(sorcery_depends_orig, sorcery_depends) + except IOError: + module.fail_json(msg="failed to copy depends.check file") + else: + sorcery_depends = os.path.join(SORCERY_STATE_DIR, "depends") + + rex = re.compile(r"^(?P\+?|\-){1}(?P[a-z0-9]+[a-z0-9_\-\+\.]*(\([A-Z0-9_\-\+\.]+\))*)$") + + for d in params['depends'].split(','): + match = rex.match(d) + + if not match: + module.fail_json(msg="wrong depends line for spell '%s'" % spell) + + # normalize status + if not match.group('status') or match.group('status') == '+': + status = 'on' + else: + status = 'off' + + depends[match.group('depend')] = status + + # drop providers spec + depends_list = [s.split('(')[0] for s in depends] + + cmd_gaze = "%s -q version %s" % (SORCERY['gaze'], ' '.join(depends_list)) + + rc, stdout, stderr = module.run_command(cmd_gaze) + + if rc != 0: + module.fail_json(msg="wrong dependencies for spell '%s'" % spell) + + fi = fileinput.input(sorcery_depends, inplace=True) + + try: + try: + for line in fi: + if line.startswith(spell + ':'): + match = None + + for d in depends: + # when local status is 'off' and dependency is provider, + # use only provider value + d_offset = d.find('(') + + if d_offset == -1: + d_p = '' + else: + d_p = re.escape(d[d_offset:]) + + # .escape() is needed mostly for the spells like 'libsigc++' + rex = re.compile("%s:(?:%s|%s):(?Pon|off):optional:" % + (re.escape(spell), re.escape(d), d_p)) + + match = rex.match(line) + + # we matched the line "spell:dependency:on|off:optional:" + if match: + # if we also matched the local status, mark dependency + # as empty and put it back into depends file + if match.group('lstatus') == depends[d]: + depends[d] = None + + sys.stdout.write(line) + + # status is not that we need, so keep this dependency + # in the list for further reverse switching; + # stop and process the next line in both cases + break + + if not match: + sys.stdout.write(line) + else: + sys.stdout.write(line) + except IOError: + module.fail_json(msg="I/O error on the depends file") + finally: + fi.close() + + depends_new = [v for v in depends if depends[v]] + + if depends_new: + try: + try: + fl = open(sorcery_depends, 'a') + + for k in depends_new: + fl.write("%s:%s:%s:optional::\n" % (spell, k, depends[k])) + except IOError: + module.fail_json(msg="I/O error on the depends file") + finally: + fl.close() + + depends_ok = False + + if module.check_mode: + try: + os.remove(sorcery_depends) + except IOError: + module.fail_json(msg="failed to clean up depends.backup file") + + return depends_ok + + +def manage_spells(module): + """ Cast or dispel spells. + + This manages the whole system ('*'), list or a single spell. Command 'cast' + is used to install or rebuild spells, while 'dispel' takes care of theirs + removal from the system. + + """ + + params = module.params + spells = params['name'] + + sorcery_queue = os.path.join(SORCERY_LOG_DIR, "queue/install") + + if spells == '*': + if params['state'] == 'latest': + # back up original queue + try: + os.rename(sorcery_queue, sorcery_queue + ".backup") + except IOError: + module.fail_json(msg="failed to backup the update queue") + + # see update_codex() + module.run_command_environ_update.update(dict(SILENT=1)) + + cmd_sorcery = "%s queue" + + rc, stdout, stderr = module.run_command(cmd_sorcery) + + if rc != 0: + module.fail_json(msg="failed to generate the update queue") + + try: + queue_size = os.stat(sorcery_queue).st_size + except: + module.fail_json(msg="failed to read the update queue") + + if queue_size != 0: + if module.check_mode: + try: + os.rename(sorcery_queue + ".backup", sorcery_queue) + except IOError: + module.fail_json(msg="failed to restore the update queue") + + module.exit_json(changed=True, msg="would have updated the system") + + cmd_cast = "%s --queue" % SORCERY['cast'] + + rc, stdout, stderr = module.run_command(cmd_cast) + + if rc != 0: + module.fail_json(msg="failed to update the system") + + module.exit_json(changed=True, msg="successfully updated the system") + else: + module.exit_json(changed=False, msg="the system is already up to date") + elif params['state'] == 'rebuild': + if module.check_mode: + module.exit_json(changed=True, msg="would have rebuilt the system") + + cmd_sorcery = "%s rebuild" % SORCERY['sorcery'] + + rc, stdout, stderr = module.run_command(cmd_sorcery) + + if rc != 0: + module.fail_json(msg="failed to rebuild the system: " + stdout) + + module.exit_json(changed=True, msg="successfully rebuilt the system") + else: + module.fail_json(msg="unsupported operation on '*' name value") + else: + if params['state'] in ('present', 'latest', 'rebuild', 'absent'): + # extract versions from the 'gaze' command + cmd_gaze = "%s -q version %s" % (SORCERY['gaze'], ' '.join(spells)) + + rc, stdout, stderr = module.run_command(cmd_gaze) + + # fail if any of spells cannot be found + if rc != 0: + module.fail_json(msg="failed to locate spell(s) in the list (%s)" % + ', '.join(spells)) + + cast_queue = [] + dispel_queue = [] + + rex = re.compile(r"[^|]+\|[^|]+\|(?P[^|]+)\|(?P[^|]+)\|(?P[^$]+)") + + # drop 2-line header and empty trailing line + for line in stdout.splitlines()[2:-1]: + match = rex.match(line) + + cast = False + + if params['state'] == 'present': + # spell is not installed.. + if match.group('inst_ver') == '-': + # ..so set up depends reqs for it + match_depends(module) + + cast = True + # spell is installed.. + else: + # ..but does not conform depends reqs + if not match_depends(module): + cast = True + elif params['state'] == 'latest': + # grimoire and installed versions do not match.. + if match.group('grim_ver') != match.group('inst_ver'): + # ..so check for depends reqs first and set them up + match_depends(module) + + cast = True + # grimoire and installed versions match.. + else: + # ..but the spell does not conform depends reqs + if not match_depends(module): + cast = True + elif params['state'] == 'rebuild': + cast = True + # 'absent' + else: + if match.group('inst_ver') != '-': + dispel_queue.append(match.group('spell')) + + if cast: + cast_queue.append(match.group('spell')) + + if cast_queue: + if module.check_mode: + module.exit_json(changed=True, msg="would have cast spell(s)") + + cmd_cast = "%s -c %s" % (SORCERY['cast'], ' '.join(cast_queue)) + + rc, stdout, stderr = module.run_command(cmd_cast) + + if rc != 0: + module.fail_json(msg="failed to cast spell(s): %s" + stdout) + + module.exit_json(changed=True, msg="successfully cast spell(s)") + elif params['state'] != 'absent': + module.exit_json(changed=False, msg="spell(s) are already cast") + + if dispel_queue: + if module.check_mode: + module.exit_json(changed=True, msg="would have dispelled spell(s)") + + cmd_dispel = "%s %s" % (SORCERY['dispel'], ' '.join(dispel_queue)) + + rc, stdout, stderr = module.run_command(cmd_dispel) + + if rc != 0: + module.fail_json(msg="failed to dispel spell(s): %s" + stdout) + + module.exit_json(changed=True, msg="successfully dispelled spell(s)") + else: + module.exit_json(changed=False, msg="spell(s) are already dispelled") + + +def main(): + module = AnsibleModule( + argument_spec = dict( + name = dict(default=None, aliases=['spell'], type='list'), + state = dict(default='present', choices=['present', 'latest', + 'absent', 'cast', 'dispelled', 'rebuild']), + depends = dict(default=None), + update = dict(default=False, choices=BOOLEANS, type='bool'), + update_cache = dict(default=False, aliases=['update_codex'], + choices=BOOLEANS, type='bool'), + cache_valid_time = dict(default=0, type='int') + ), + required_one_of = [['name', 'update', 'update_cache']], + supports_check_mode = True + ) + + if os.geteuid() != 0: + module.fail_json(msg="root privileges are required for this operation") + + for c in SORCERY: + SORCERY[c] = module.get_bin_path(c, True) + + # prepare environment: run sorcery commands without asking questions + module.run_command_environ_update = dict(PROMPT_DELAY=0, VOYEUR=0) + + params = module.params + + # normalize 'state' parameter + if params['state'] in ('present', 'cast'): + params['state'] = 'present' + elif params['state'] in ('absent', 'dispelled'): + params['state'] = 'absent' + + if params['update']: + update_sorcery(module) + + if params['update_cache'] or params['state'] == 'latest': + update_codex(module) + + if params['name']: + manage_spells(module) + + +# import module snippets +from ansible.module_utils.basic import AnsibleModule + +if __name__ == '__main__': + main()