cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

scan_packages: made adding package managers easier (#49079)

Browse source 
* made adding package managers easier

  added portage support

* moar pkg mgrs and moar info

 - added 'pkg' pkg manager (freebsd)
 - added pip
 - more apt info

* updated clgo

* Updates from feedback

Co-Authored-By: bcoca <bcoca@users.noreply.github.com>

* incorporated more feedback and added docstrings

* moar from feedback

  - made manager list dynamic and names based on class
  - better not found msg
  - made abstract metaclass again
  - test is now init exception
  - module to global
  - better dedupe comments

* more targetted errors/warnings

* added strategy, reordered to conserve priority

* rpm > apt

* move break to top

* fix trate

* piping it

* lines and meta

* refactored common functions

 - moved pip into it's own module
 - cleaned up base clases
 - ensure 'lower' match in package_facts

* missing license

* avoid facts

* update clog

* addressed feedback

* fix clog

* cleanup

* upd

* removed pip as that was removed

* renamed cpan

* added a single line since 2 lines are needed to be

readabnle instead of just 1 line, it is a huge problem otherwise

* fix internal ref

* not intended in this round

* updated as per fb
This commit is contained in:
Brian Coca 2019-03-06 12:29:51 -05:00 committed by GitHub
parent 3fe2013b3f
commit b94bfed1a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 393 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
changelogs/fragments/scan_packages.yml Normal file
View file

@ -0,0 +1,3 @@
minor_changes:
- package_facts, now supports multiple package managers per system.
New systems supported include Gentoo's portage with portage-utils installed, as well as FreeBSD's pkg

83
lib/ansible/module_utils/facts/packages.py Normal file
View file

@ -0,0 +1,83 @@
# (c) 2018, Ansible Project
# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
from abc import ABCMeta, abstractmethod
from ansible.module_utils.six import with_metaclass
from ansible.module_utils.basic import get_all_subclasses
from ansible.module_utils.common.process import get_bin_path
def get_all_pkg_managers():
return dict([(obj.__name__.lower(), obj) for obj in get_all_subclasses(PkgMgr) if obj not in (CLIMgr, LibMgr)])
class PkgMgr(with_metaclass(ABCMeta, object)):
@abstractmethod
def is_available(self):
# This method is supposed to return True/False if the package manager is currently installed/usable
# It can also 'prep' the required systems in the process of detecting availability
pass
@abstractmethod
def list_installed(self):
# This method should return a list of installed packages, each list item will be passed to get_package_details
pass
@abstractmethod
def get_package_details(self, package):
# This takes a 'package' item and returns a dictionary with the package information, name and version are minimal requirements
pass
def get_packages(self):
# Take all of the above and return a dictionary of lists of dictionaries (package = list of installed versions)
installed_packages = {}
for package in self.list_installed():
package_details = self.get_package_details(package)
if 'source' not in package_details:
package_details['source'] = self.__class__.__name__.lower()
name = package_details['name']
if name not in installed_packages:
installed_packages[name] = [package_details]
else:
installed_packages[name].append(package_details)
return installed_packages
class LibMgr(PkgMgr):
LIB = None
def __init__(self):
self._lib = None
super(LibMgr, self).__init__()
def is_available(self):
found = False
try:
self._lib = __import__(self.LIB)
found = True
except ImportError:
pass
return found
class CLIMgr(PkgMgr):
CLI = None
def __init__(self):
self._cli = None
super(CLIMgr, self).__init__()
def is_available(self):
self._cli = get_bin_path(self.CLI, False)
return bool(self._cli)

152
lib/ansible/modules/packaging/language/pip_package_info.py Normal file
View file

@ -0,0 +1,152 @@
#!/usr/bin/python
# (c) 2018, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# started out with AWX's scan_packages module
from __future__ import absolute_import, division, print_function
__metaclass__ = type
ANSIBLE_METADATA = {'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'}
DOCUMENTATION = '''
module: pip_package_info
short_description: pip package information
description:
- Return information about installed pip packages
version_added: "2.8"
options:
clients:
description:
- A list of the pip executables that will be used to get the packages.
They can be supplied with the full path or just the executable name, i.e `pip3.7`.
default: ['pip']
required: False
type: list
requirements:
- The requested pip executables must be installed on the target.
author:
- Matthew Jones (@matburt)
- Brian Coca (@bcoca)
- Adam Miller (@maxamillion)
'''
EXAMPLES = '''
- name: Just get the list from default pip
pip_package_info:
- name: get the facts for default pip, pip2 and pip3.6
pip_package_info:
clients: ['pip', 'pip2', 'pip3.6']
- name: get from specific paths (virtualenvs?)
pip_package_info:
clients: '/home/me/projec42/python/pip3.5'
'''
RETURN = '''
packages:
description: a dictionary of installed package data
returned: always
type: dict
contains:
python:
description: A dictionary with each pip client which then contains a list of dicts with python package information
returned: always
type: dict
sample:
"packages": {
"pip": {
"Babel": [
{
"name": "Babel",
"source": "pip",
"version": "2.6.0"
}
],
"Flask": [
{
"name": "Flask",
"source": "pip",
"version": "1.0.2"
}
],
"Flask-SQLAlchemy": [
{
"name": "Flask-SQLAlchemy",
"source": "pip",
"version": "2.3.2"
}
],
"Jinja2": [
{
"name": "Jinja2",
"source": "pip",
"version": "2.10"
}
],
},
}
'''
import json
import os
from ansible.module_utils._text import to_text
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.facts.packages import CLIMgr
class PIP(CLIMgr):
def __init__(self, pip):
self.CLI = pip
def list_installed(self):
global module
rc, out, err = module.run_command([self._cli, 'list', '-l', '--format=json'])
if rc != 0:
raise Exception("Unable to list packages rc=%s : %s" % (rc, err))
return json.loads(out)
def get_package_details(self, package):
package['source'] = self.CLI
return package
def main():
# start work
global module
module = AnsibleModule(argument_spec=dict(clients={'type': 'list', 'default': ['pip']},), supports_check_mode=True)
packages = {}
results = {'packages': {}}
clients = module.params['clients']
found = 0
for pip in clients:
if not os.path.basename(pip).startswith('pip'):
module.warn('Skipping invalid pip client: %s' % (pip))
continue
try:
pip_mgr = PIP(pip)
if pip_mgr.is_available():
found += 1
packages[pip] = pip_mgr.get_packages()
except Exception as e:
module.warn('Failed to retrieve packages with %s: %s' % (pip, to_text(e)))
continue
if found == 0:
module.fail_json(msg='Unable to use any of the supplied pip clients: %s' % clients)
# return info
results['packages'] = packages
module.exit_json(**results)
if __name__ == '__main__':
main()

229
lib/ansible/modules/packaging/os/package_facts.py
View file

@ -19,11 +19,24 @@ description:
options:
manager:
description:
- The package manager used by the system so we can query the package information
default: auto
choices: ["auto", "rpm", "apt"]
- The package manager used by the system so we can query the package information.
- Since 2.8 this is a list and can support multiple package managers per system.
- The 'portage' and 'pkg' options were added in version 2.8.
default: ['auto']
choices: ['auto', 'rpm', 'apt', 'portage', 'pkg']
required: False
type: list
strategy:
description:
- This option controls how the module queres the package managers on the system.
C(first) means it will return only informatino for the first supported package manager available.
C(all) will return information for all supported and available package managers on the system.
choices: ['first', 'all']
default: 'first'
version_added: "2.8"
version_added: "2.5"
requirements:
- For 'portage' support it requires the `qlist` utility, which is part of 'app-portage/portage-utils'.
author:
- Matthew Jones (@matburt)
- Brian Coca (@bcoca)
@ -140,94 +153,162 @@ ansible_facts:
}
'''
import sys
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils._text import to_text
from ansible.module_utils.facts.packages import LibMgr, CLIMgr, get_all_pkg_managers
def rpm_package_list():
class RPM(LibMgr):
try:
import rpm
except ImportError:
module.fail_json(msg='Unable to use the rpm python bindings, please ensure they are installed under the python the module runs under')
LIB = 'rpm'
trans_set = rpm.TransactionSet()
installed_packages = {}
for package in trans_set.dbMatch():
package_details = dict(name=package[rpm.RPMTAG_NAME],
version=package[rpm.RPMTAG_VERSION],
release=package[rpm.RPMTAG_RELEASE],
epoch=package[rpm.RPMTAG_EPOCH],
arch=package[rpm.RPMTAG_ARCH],
source='rpm')
if package_details['name'] not in installed_packages:
installed_packages[package_details['name']] = [package_details]
else:
installed_packages[package_details['name']].append(package_details)
return installed_packages
def list_installed(self):
return self._lib.TransactionSet().dbMatch()
def get_package_details(self, package):
return dict(name=package[self._lib.RPMTAG_NAME],
version=package[self._lib.RPMTAG_VERSION],
release=package[self._lib.RPMTAG_RELEASE],
epoch=package[self._lib.RPMTAG_EPOCH],
arch=package[self._lib.RPMTAG_ARCH],)
def apt_package_list():
class APT(LibMgr):
try:
import apt
except ImportError:
module.fail_json(msg='Unable to use the apt python bindings, please ensure they are installed under the python the module runs under')
LIB = 'apt'
apt_cache = apt.Cache()
installed_packages = {}
apt_installed_packages = [pk for pk in apt_cache.keys() if apt_cache[pk].is_installed]
for package in apt_installed_packages:
ac_pkg = apt_cache[package].installed
package_details = dict(name=package, version=ac_pkg.version, arch=ac_pkg.architecture, source='apt')
if package_details['name'] not in installed_packages:
installed_packages[package_details['name']] = [package_details]
else:
installed_packages[package_details['name']].append(package_details)
return installed_packages
@property
def pkg_cache(self):
if self._cache:
return self._cache
self._cache = self._lib.Cache()
return self._cache
def list_installed(self):
return [pk for pk in self.pkg_cache.keys() if self.pkg_cache[pk].is_installed]
def get_package_details(self, package):
ac_pkg = self.pkg_cache[package].installed
return dict(name=package, version=ac_pkg.version, arch=ac_pkg.architecture, category=ac_pkg.section, origin=ac_pkg.origins[0].origin)
# FIXME: add more listing methods
def main():
global module
module = AnsibleModule(argument_spec=dict(manager=dict()), supports_check_mode=True)
manager = module.params['manager']
packages = {}
results = {}
class PKG(CLIMgr):
if manager is None or manager == 'auto':
CLI = 'pkg'
atoms = ['name', 'version', 'origin', 'installed', 'automatic', 'arch', 'category', 'prefix', 'vital']
# detect!
for manager_lib in ('rpm', 'apt'):
def list_installed(self):
rc, out, err = module.run_command([self._cli, 'query', "%%%s" % '\t%'.join(['n', 'v', 'R', 't', 'a', 'q', 'o', 'p', 'V'])])
if rc != 0 or err:
raise Exception("Unable to list packages rc=%s : %s" % (rc, err))
return out.splitlines()
def get_package_details(self, package):
pkg = dict(zip(self.atoms, package.split('\t')))
if 'arch' in pkg:
try:
dummy = __import__(manager_lib)
manager = manager_lib
break
except ImportError:
pkg['arch'] = pkg['arch'].split(':')[2]
except IndexError:
pass
# FIXME: add more detection methods
try:
if manager == "rpm":
packages = rpm_package_list()
elif manager == "apt":
packages = apt_package_list()
else:
if manager:
results['msg'] = 'Unsupported package manager: %s' % manager
results['skipped'] = True
else:
module.fail_json(msg='Could not detect supported package manager')
except Exception as e:
from traceback import format_tb
module.fail_json(msg='Failed to retrieve packages: %s' % to_text(e), exception=format_tb(sys.exc_info()[2]))
if 'automatic' in pkg:
pkg['automatic'] = bool(pkg['automatic'])
results['ansible_facts'] = {}
# Set the facts, this will override the facts in ansible_facts that might
# exist from previous runs when using operating system level or distribution
# package managers
if 'category' in pkg:
pkg['category'] = pkg['category'].split('/', 1)[0]
if 'version' in pkg:
if ',' in pkg['version']:
pkg['version'], pkg['port_epoch'] = pkg['version'].split(',', 1)
else:
pkg['port_epoch'] = 0
if '_' in pkg['version']:
pkg['version'], pkg['revision'] = pkg['version'].split('_', 1)
else:
pkg['revision'] = '0'
if 'vital' in pkg:
pkg['vital'] = bool(pkg['vital'])
return pkg
class PORTAGE(CLIMgr):
CLI = 'qlist'
atoms = ['category', 'name', 'version', 'ebuild_revision', 'slots', 'prefixes', 'sufixes']
def list_installed(self):
rc, out, err = module.run_command(' '.join([self._cli, '-Iv', '|', 'xargs', '-n', '1024', 'qatom']), use_unsafe_shell=True)
if rc != 0:
raise RuntimeError("Unable to list packages rc=%s : %s" % (rc, to_native(err)))
return out.splitlines()
def get_package_details(self, package):
return dict(zip(self.atoms, package.split()))
def main():
# get supported pkg managers
PKG_MANAGERS = get_all_pkg_managers()
PKG_MANAGER_NAMES = [x.lower() for x in PKG_MANAGERS.keys()]
# start work
global module
module = AnsibleModule(argument_spec=dict(manager={'type': 'list', 'default': ['auto']},
strategy={'choices': ['first', 'all'], 'default': 'first'}),
supports_check_mode=True)
packages = {}
results = {'ansible_facts': {}}
managers = [x.lower() for x in module.params['manager']]
strategy = module.params['strategy']
if 'auto' in managers:
# keep order from user, we do dedupe below
managers.extend(PKG_MANAGER_NAMES)
managers.remove('auto')
unsupported = set(managers).difference(PKG_MANAGER_NAMES)
if unsupported:
module.fail_json(msg='Unsupported package managers requested: %s' % (', '.join(unsupported)))
found = 0
seen = set()
for pkgmgr in managers:
if found and strategy == 'first':
break
# dedupe as per above
if pkgmgr in seen:
continue
seen.add(pkgmgr)
try:
try:
# manager throws exception on init (calls self.test) if not usable.
manager = PKG_MANAGERS[pkgmgr]()
if manager.is_available():
found += 1
packages.update(manager.get_packages())
except Exception as e:
if pkgmgr in module.params['manager']:
module.warn('Requested package manager %s was not usable by this module: %s' % (pkgmgr, to_text(e)))
continue
except Exception as e:
if pkgmgr in module.params['manager']:
module.warn('Failed to retrieve packages with %s: %s' % (pkgmgr, to_text(e)))
if found == 0:
module.fail_json(msg='Could not detect a supported package manager from the following list: %s' % managers)
# Set the facts, this will override the facts in ansible_facts that might exist from previous runs
# when using operating system level or distribution package managers
results['ansible_facts']['packages'] = packages
module.exit_json(**results)

1
test/sanity/validate-modules/ignore.txt
View file

@ -721,7 +721,6 @@ lib/ansible/modules/packaging/os/opkg.py E322
lib/ansible/modules/packaging/os/opkg.py E324
lib/ansible/modules/packaging/os/opkg.py E326
lib/ansible/modules/packaging/os/opkg.py E336
lib/ansible/modules/packaging/os/package_facts.py E324
lib/ansible/modules/packaging/os/package_facts.py E326
lib/ansible/modules/packaging/os/pacman.py E326
lib/ansible/modules/packaging/os/pacman.py E336