ansible/test/sanity/code-smell/update-bundled.py
Matt Martz 8d1cf7f266
Vendor distutils.version ()
* Vendor distutils.version

* Fix import order. ci_complete

* remove distutils warning filter

* Don't remove warnings filter from importer

* ci_complete

* Add pylint config for preventing distutils.version

* Add changelog fragment
2021-05-11 12:33:51 -05:00

177 lines
6.1 KiB
Python
Executable file

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# (c) 2018, Ansible Project
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""
This test checks whether the libraries we're bundling are out of date and need to be synced with
a newer upstream release.
"""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import fnmatch
import json
import re
import sys
from ansible.module_utils.compat.version import LooseVersion
import packaging.specifiers
from ansible.module_utils.urls import open_url
BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b')
def get_bundled_libs(paths):
"""
Return the set of known bundled libraries
:arg paths: The paths which the test has been instructed to check
:returns: The list of all files which we know to contain bundled libraries. If a bundled
library consists of multiple files, this should be the file which has metadata included.
"""
bundled_libs = set()
for filename in fnmatch.filter(paths, 'lib/ansible/compat/*/__init__.py'):
bundled_libs.add(filename)
bundled_libs.add('lib/ansible/module_utils/compat/selectors.py')
bundled_libs.add('lib/ansible/module_utils/distro/__init__.py')
bundled_libs.add('lib/ansible/module_utils/six/__init__.py')
# backports.ssl_match_hostname should be moved to its own file in the future
bundled_libs.add('lib/ansible/module_utils/urls.py')
return bundled_libs
def get_files_with_bundled_metadata(paths):
"""
Search for any files which have bundled metadata inside of them
:arg paths: Iterable of filenames to search for metadata inside of
:returns: A set of pathnames which contained metadata
"""
with_metadata = set()
for path in paths:
with open(path, 'rb') as f:
body = f.read()
if BUNDLED_RE.search(body):
with_metadata.add(path)
return with_metadata
def get_bundled_metadata(filename):
"""
Retrieve the metadata about a bundled library from a python file
:arg filename: The filename to look inside for the metadata
:raises ValueError: If we're unable to extract metadata from the file
:returns: The metadata from the python file
"""
with open(filename, 'r') as module:
for line in module:
if line.strip().startswith('# NOT_BUNDLED'):
return None
if line.strip().startswith('# CANT_UPDATE'):
print(
'{0} marked as CANT_UPDATE, so skipping. Manual '
'check for CVEs required.'.format(filename))
return None
if line.strip().startswith('_BUNDLED_METADATA'):
data = line[line.index('{'):].strip()
break
else:
raise ValueError('Unable to check bundled library for update. Please add'
' _BUNDLED_METADATA dictionary to the library file with'
' information on pypi name and bundled version.')
metadata = json.loads(data)
return metadata
def get_latest_applicable_version(pypi_data, constraints=None):
"""Get the latest pypi version of the package that we allow
:arg pypi_data: Pypi information about the data as returned by
``https://pypi.org/pypi/{pkg_name}/json``
:kwarg constraints: version constraints on what we're allowed to use as specified by
the bundled metadata
:returns: The most recent version on pypi that are allowed by ``constraints``
"""
latest_version = "0"
if constraints:
version_specification = packaging.specifiers.SpecifierSet(constraints)
for version in pypi_data['releases']:
if version in version_specification:
if LooseVersion(version) > LooseVersion(latest_version):
latest_version = version
else:
latest_version = pypi_data['info']['version']
return latest_version
def main():
"""Entrypoint to the script"""
paths = sys.argv[1:] or sys.stdin.read().splitlines()
bundled_libs = get_bundled_libs(paths)
files_with_bundled_metadata = get_files_with_bundled_metadata(paths)
for filename in files_with_bundled_metadata.difference(bundled_libs):
print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to'
' test/sanity/code-smell/update-bundled.py'.format(filename))
for filename in bundled_libs:
try:
metadata = get_bundled_metadata(filename)
except ValueError as e:
print('{0}: ERROR: {1}'.format(filename, e))
continue
except (IOError, OSError) as e:
if e.errno == 2:
print('{0}: ERROR: {1}. Perhaps the bundled library has been removed'
' or moved and the bundled library test needs to be modified as'
' well?'.format(filename, e))
if metadata is None:
continue
pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
pypi_data = json.loads(pypi_fh.read().decode('utf-8'))
constraints = metadata.get('version_constraints', None)
latest_version = get_latest_applicable_version(pypi_data, constraints)
if LooseVersion(metadata['version']) < LooseVersion(latest_version):
print('{0}: UPDATE {1} from {2} to {3} {4}'.format(
filename,
metadata['pypi_name'],
metadata['version'],
latest_version,
'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))
if __name__ == '__main__':
main()