Have update-bundled check for updates to all bundled code
Add a test for whether we have bundled code inside of ansible that needs to be updated
This commit is contained in:
parent
2c0897caf2
commit
dfd8b659c0
8 changed files with 184 additions and 35 deletions
31
docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst
Normal file
31
docs/docsite/rst/dev_guide/testing/sanity/update-bundled.rst
Normal file
|
@ -0,0 +1,31 @@
|
|||
:orphan:
|
||||
|
||||
Sanity Tests » update-bundled
|
||||
=============================
|
||||
|
||||
Check whether any of our known bundled code needs to be updated for a new upstream release.
|
||||
|
||||
This test can error in the following ways:
|
||||
|
||||
* The bundled code is out of date with regard to the latest release on pypi. Update the code
|
||||
to the new version and update the version in _BUNDLED_METADATA to solve this.
|
||||
|
||||
* The code is lacking a _BUNDLED_METADATA variable. This typically happens when a bundled version
|
||||
is updated and we forget to add a _BUNDLED_METADATA variable to the updated file. Once that is
|
||||
added, this error should go away.
|
||||
|
||||
* A file has a _BUNDLED_METADATA variable but the file isn't specified in
|
||||
:file:`test/sanity/code-smell/update-bundled.py`. This typically happens when a new bundled
|
||||
library is added. Add the file to the `get_bundled_libs()` function in the `update-bundled.py`
|
||||
test script to solve this error.
|
||||
|
||||
_BUNDLED_METADATA has the following fields:
|
||||
|
||||
:pypi_name: Name of the bundled package on pypi
|
||||
|
||||
:version: Version of the package that we are including here
|
||||
|
||||
:version_constraints: Optional PEP440 specifier for the version range that we are bundling.
|
||||
Currently, the only valid use of this is to follow a version that is
|
||||
compatible with the Python stdlib when newer versions of the pypi package
|
||||
implement a new API.
|
|
@ -1,34 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os.path
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from ansible.module_utils.urls import open_url
|
||||
|
||||
basedir = os.path.dirname(__file__)
|
||||
|
||||
for filename in glob.glob(os.path.join(basedir, '../lib/ansible/compat/*/__init__.py')):
|
||||
if 'compat/tests' in filename:
|
||||
# compat/tests doesn't bundle any code
|
||||
continue
|
||||
|
||||
filename = os.path.normpath(filename)
|
||||
with open(filename, 'r') as module:
|
||||
for line in module:
|
||||
if line.strip().startswith('_BUNDLED_METADATA'):
|
||||
data = line[line.index('{'):].strip()
|
||||
break
|
||||
else:
|
||||
print('WARNING: {0} contained no metadata. Could not check for updates'.format(filename))
|
||||
continue
|
||||
metadata = json.loads(data)
|
||||
pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
|
||||
pypi_data = json.loads(pypi_fh.read().decode('utf-8'))
|
||||
if LooseVersion(metadata['version']) < LooseVersion(pypi_data['info']['version']):
|
||||
print('UPDATE: {0} from {1} to {2} {3}'.format(
|
||||
metadata['pypi_name'],
|
||||
metadata['version'],
|
||||
pypi_data['info']['version'],
|
||||
'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))
|
|
@ -24,7 +24,8 @@ Compat selectors library. Python-3.5 has this builtin. The selectors2
|
|||
package exists on pypi to backport the functionality as far as python-2.6.
|
||||
'''
|
||||
# The following makes it easier for us to script updates of the bundled code
|
||||
_BUNDLED_METADATA = {"pypi_name": "selectors2", "version": "1.1.0"}
|
||||
_BUNDLED_METADATA = {"pypi_name": "selectors2", "version": "1.1.0", "version_constraints": ">1.0,<2.0"}
|
||||
|
||||
# Added these bugfix commits from 2.1.0:
|
||||
# * https://github.com/SethMichaelLarson/selectors2/commit/3bd74f2033363b606e1e849528ccaa76f5067590
|
||||
# Wrap kqueue.control so that timeout is a keyword arg
|
||||
|
|
|
@ -66,6 +66,11 @@ from __future__ import unicode_literals
|
|||
import itertools
|
||||
import struct
|
||||
|
||||
|
||||
# The following makes it easier for us to script updates of the bundled code and is not part of
|
||||
# upstream
|
||||
_BUNDLED_METADATA = {"pypi_name": "ipaddress", "version": "1.0.22"}
|
||||
|
||||
__version__ = '1.0.22'
|
||||
|
||||
# Compatibility functions
|
||||
|
|
|
@ -33,6 +33,10 @@ import operator
|
|||
import sys
|
||||
import types
|
||||
|
||||
# The following makes it easier for us to script updates of the bundled code. It is not part of
|
||||
# upstream six
|
||||
_BUNDLED_METADATA = {"pypi_name": "six", "version": "1.11.0"}
|
||||
|
||||
__author__ = "Benjamin Peterson <benjamin@python.org>"
|
||||
__version__ = "1.11.0"
|
||||
|
||||
|
|
|
@ -136,6 +136,10 @@ if not HAS_SSLCONTEXT and HAS_SSL:
|
|||
del libssl
|
||||
|
||||
|
||||
# The following makes it easier for us to script updates of the bundled backports.ssl_match_hostname
|
||||
# The bundled backports.ssl_match_hostname should really be moved into its own file for processing
|
||||
_BUNDLED_METADATA = {"pypi_name": "backports.ssl_match_hostname", "version": "3.5.0.1"}
|
||||
|
||||
LOADED_VERIFY_LOCATIONS = set()
|
||||
|
||||
HAS_MATCH_HOSTNAME = True
|
||||
|
|
7
test/sanity/code-smell/update-bundled.json
Normal file
7
test/sanity/code-smell/update-bundled.json
Normal file
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"ignore_changes": true,
|
||||
"extensions": [
|
||||
".py"
|
||||
],
|
||||
"output": "path-message"
|
||||
}
|
131
test/sanity/code-smell/update-bundled.py
Executable file
131
test/sanity/code-smell/update-bundled.py
Executable file
|
@ -0,0 +1,131 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# (c) 2018, Ansible Project
|
||||
#
|
||||
# This file is part of Ansible
|
||||
#
|
||||
# Ansible is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Ansible is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
|
||||
import fnmatch
|
||||
import json
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
import packaging.specifiers
|
||||
|
||||
from ansible.module_utils.urls import open_url
|
||||
|
||||
|
||||
BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b')
|
||||
|
||||
|
||||
def get_bundled_libs(paths):
|
||||
bundled_libs = set()
|
||||
for filename in fnmatch.filter(paths, 'lib/ansible/compat/*/__init__.py'):
|
||||
bundled_libs.add(filename)
|
||||
|
||||
bundled_libs.add('lib/ansible/module_utils/distro/__init__.py')
|
||||
bundled_libs.add('lib/ansible/module_utils/six/__init__.py')
|
||||
bundled_libs.add('lib/ansible/module_utils/compat/ipaddress.py')
|
||||
# backports.ssl_match_hostname should be moved to its own file in the future
|
||||
bundled_libs.add('lib/ansible/module_utils/urls.py')
|
||||
|
||||
return bundled_libs
|
||||
|
||||
|
||||
def get_files_with_bundled_metadata(paths):
|
||||
with_metadata = set()
|
||||
for path in paths:
|
||||
if path == 'test/sanity/code-smell/update-bundled.py':
|
||||
continue
|
||||
|
||||
with open(path, 'rb') as f:
|
||||
body = f.read()
|
||||
|
||||
if BUNDLED_RE.search(body):
|
||||
with_metadata.add(path)
|
||||
|
||||
return with_metadata
|
||||
|
||||
|
||||
def get_bundled_metadata(filename):
|
||||
with open(filename, 'r') as module:
|
||||
for line in module:
|
||||
if line.strip().startswith('_BUNDLED_METADATA'):
|
||||
data = line[line.index('{'):].strip()
|
||||
break
|
||||
else:
|
||||
raise ValueError('Unable to check bundled library for update. Please add'
|
||||
' _BUNDLED_METADATA dictionary to the library file with'
|
||||
' information on pypi name and bundled version.')
|
||||
metadata = json.loads(data)
|
||||
return metadata
|
||||
|
||||
|
||||
def get_latest_applicable_version(pypi_data, constraints=None):
|
||||
latest_version = "0"
|
||||
if 'version_constraints' in metadata:
|
||||
version_specification = packaging.specifiers.SpecifierSet(metadata['version_constraints'])
|
||||
for version in pypi_data['releases']:
|
||||
if version in version_specification:
|
||||
if LooseVersion(version) > LooseVersion(latest_version):
|
||||
latest_version = version
|
||||
else:
|
||||
latest_version = pypi_data['info']['version']
|
||||
|
||||
return latest_version
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
paths = sys.argv[1:] or sys.stdin.read().splitlines()
|
||||
|
||||
bundled_libs = get_bundled_libs(paths)
|
||||
files_with_bundled_metadata = get_files_with_bundled_metadata(paths)
|
||||
|
||||
for filename in files_with_bundled_metadata.difference(bundled_libs):
|
||||
print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to'
|
||||
' test/sanity/code-smell/update-bundled.py'.format(filename))
|
||||
|
||||
for filename in bundled_libs:
|
||||
try:
|
||||
metadata = get_bundled_metadata(filename)
|
||||
except ValueError as e:
|
||||
print('{0}: ERROR: {1}'.format(filename, e))
|
||||
continue
|
||||
except (IOError, OSError) as e:
|
||||
if e.errno == 2:
|
||||
print('{0}: ERROR: {1}. Perhaps the bundled library has been removed'
|
||||
' or moved and the bundled library test needs to be modified as'
|
||||
' well?'.format(filename, e))
|
||||
|
||||
pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
|
||||
pypi_data = json.loads(pypi_fh.read().decode('utf-8'))
|
||||
|
||||
constraints = metadata.get('version_constraints', None)
|
||||
latest_version = get_latest_applicable_version(pypi_data, constraints)
|
||||
|
||||
if LooseVersion(metadata['version']) < LooseVersion(latest_version):
|
||||
print('{0}: UPDATE {1} from {2} to {3} {4}'.format(
|
||||
filename,
|
||||
metadata['pypi_name'],
|
||||
metadata['version'],
|
||||
latest_version,
|
||||
'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))
|
Loading…
Reference in a new issue