92b66e3e31
* Make Azure Pipelines resource_prefix lowercase. * Make classification of CI files consistent. * Update package-data sanity test for AZP.
376 lines
12 KiB
Python
Executable file
376 lines
12 KiB
Python
Executable file
#!/usr/bin/env python
|
|
from __future__ import (absolute_import, division, print_function)
|
|
__metaclass__ = type
|
|
|
|
import contextlib
|
|
import fnmatch
|
|
import glob
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
|
|
|
|
def assemble_files_to_ship(complete_file_list):
|
|
"""
|
|
This looks for all files which should be shipped in the sdist
|
|
"""
|
|
# All files which are in the repository except these:
|
|
ignore_patterns = (
|
|
# Developer-only tools
|
|
'.azure-pipelines/*',
|
|
'.github/*',
|
|
'.github/*/*',
|
|
'changelogs/fragments/*',
|
|
'hacking/backport/*',
|
|
'hacking/shippable/*',
|
|
'hacking/tests/*',
|
|
'hacking/ticket_stubs/*',
|
|
'test/sanity/code-smell/botmeta.*',
|
|
'test/utils/*',
|
|
'test/utils/*/*',
|
|
'test/utils/*/*/*',
|
|
'.git*',
|
|
)
|
|
ignore_files = frozenset((
|
|
# Developer-only tools
|
|
'changelogs/config.yaml',
|
|
'hacking/README.md',
|
|
'hacking/ansible-profile',
|
|
'hacking/cgroup_perf_recap_graph.py',
|
|
'hacking/create_deprecated_issues.py',
|
|
'hacking/deprecated_issue_template.md',
|
|
'hacking/fix_test_syntax.py',
|
|
'hacking/get_library.py',
|
|
'hacking/metadata-tool.py',
|
|
'hacking/report.py',
|
|
'hacking/return_skeleton_generator.py',
|
|
'hacking/test-module',
|
|
'hacking/test-module.py',
|
|
'test/support/README.md',
|
|
'.cherry_picker.toml',
|
|
'.mailmap',
|
|
# Possibly should be included
|
|
'examples/scripts/uptime.py',
|
|
'examples/scripts/my_test.py',
|
|
'examples/scripts/my_test_info.py',
|
|
'examples/scripts/my_test_facts.py',
|
|
'examples/DOCUMENTATION.yml',
|
|
'examples/play.yml',
|
|
'examples/hosts.yaml',
|
|
'examples/hosts.yml',
|
|
'examples/inventory_script_schema.json',
|
|
'examples/plugin_filters.yml',
|
|
'hacking/env-setup',
|
|
'hacking/env-setup.fish',
|
|
'MANIFEST',
|
|
))
|
|
|
|
# These files are generated and then intentionally added to the sdist
|
|
|
|
# Manpages
|
|
manpages = ['docs/man/man1/ansible.1']
|
|
for dirname, dummy, files in os.walk('bin'):
|
|
for filename in files:
|
|
path = os.path.join(dirname, filename)
|
|
if os.path.islink(path):
|
|
if os.readlink(path) == 'ansible':
|
|
manpages.append('docs/man/man1/%s.1' % filename)
|
|
|
|
# Misc
|
|
misc_generated_files = [
|
|
'SYMLINK_CACHE.json',
|
|
'PKG-INFO',
|
|
]
|
|
|
|
shipped_files = manpages + misc_generated_files
|
|
|
|
for path in complete_file_list:
|
|
if path not in ignore_files:
|
|
for ignore in ignore_patterns:
|
|
if fnmatch.fnmatch(path, ignore):
|
|
break
|
|
else:
|
|
shipped_files.append(path)
|
|
|
|
return shipped_files
|
|
|
|
|
|
def assemble_files_to_install(complete_file_list):
|
|
"""
|
|
This looks for all of the files which should show up in an installation of ansible
|
|
"""
|
|
ignore_patterns = tuple()
|
|
|
|
pkg_data_files = []
|
|
for path in complete_file_list:
|
|
|
|
if path.startswith("lib/ansible"):
|
|
prefix = 'lib'
|
|
elif path.startswith("test/lib/ansible_test"):
|
|
prefix = 'test/lib'
|
|
else:
|
|
continue
|
|
|
|
for ignore in ignore_patterns:
|
|
if fnmatch.fnmatch(path, ignore):
|
|
break
|
|
else:
|
|
pkg_data_files.append(os.path.relpath(path, prefix))
|
|
|
|
return pkg_data_files
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def clean_repository(file_list):
|
|
"""Copy the repository to clean it of artifacts"""
|
|
# Create a tempdir that will be the clean repo
|
|
with tempfile.TemporaryDirectory() as repo_root:
|
|
directories = set((repo_root + os.path.sep,))
|
|
|
|
for filename in file_list:
|
|
# Determine if we need to create the directory
|
|
directory = os.path.dirname(filename)
|
|
dest_dir = os.path.join(repo_root, directory)
|
|
if dest_dir not in directories:
|
|
os.makedirs(dest_dir)
|
|
|
|
# Keep track of all the directories that now exist
|
|
path_components = directory.split(os.path.sep)
|
|
path = repo_root
|
|
for component in path_components:
|
|
path = os.path.join(path, component)
|
|
if path not in directories:
|
|
directories.add(path)
|
|
|
|
# Copy the file
|
|
shutil.copy2(filename, dest_dir, follow_symlinks=False)
|
|
|
|
yield repo_root
|
|
|
|
|
|
def create_sdist(tmp_dir):
|
|
"""Create an sdist in the repository"""
|
|
create = subprocess.Popen(
|
|
['make', 'snapshot', 'SDIST_DIR=%s' % tmp_dir],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
universal_newlines=True,
|
|
)
|
|
|
|
stderr = create.communicate()[1]
|
|
|
|
if create.returncode != 0:
|
|
raise Exception('make snapshot failed:\n%s' % stderr)
|
|
|
|
# Determine path to sdist
|
|
tmp_dir_files = os.listdir(tmp_dir)
|
|
|
|
if not tmp_dir_files:
|
|
raise Exception('sdist was not created in the temp dir')
|
|
elif len(tmp_dir_files) > 1:
|
|
raise Exception('Unexpected extra files in the temp dir')
|
|
|
|
return os.path.join(tmp_dir, tmp_dir_files[0])
|
|
|
|
|
|
def extract_sdist(sdist_path, tmp_dir):
|
|
"""Untar the sdist"""
|
|
# Untar the sdist from the tmp_dir
|
|
with tarfile.open(os.path.join(tmp_dir, sdist_path), 'r|*') as sdist:
|
|
sdist.extractall(path=tmp_dir)
|
|
|
|
# Determine the sdist directory name
|
|
sdist_filename = os.path.basename(sdist_path)
|
|
tmp_dir_files = os.listdir(tmp_dir)
|
|
try:
|
|
tmp_dir_files.remove(sdist_filename)
|
|
except ValueError:
|
|
# Unexpected could not find original sdist in temp dir
|
|
raise
|
|
|
|
if len(tmp_dir_files) > 1:
|
|
raise Exception('Unexpected extra files in the temp dir')
|
|
elif len(tmp_dir_files) < 1:
|
|
raise Exception('sdist extraction did not occur i nthe temp dir')
|
|
|
|
return os.path.join(tmp_dir, tmp_dir_files[0])
|
|
|
|
|
|
def install_sdist(tmp_dir, sdist_dir):
|
|
"""Install the extracted sdist into the temporary directory"""
|
|
install = subprocess.Popen(
|
|
['python', 'setup.py', 'install', '--root=%s' % tmp_dir],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
universal_newlines=True,
|
|
cwd=os.path.join(tmp_dir, sdist_dir),
|
|
)
|
|
|
|
stdout, stderr = install.communicate()
|
|
|
|
if install.returncode != 0:
|
|
raise Exception('sdist install failed:\n%s' % stderr)
|
|
|
|
# Determine the prefix for the installed files
|
|
match = re.search('^creating (%s/.*?/(?:site|dist)-packages)/ansible$' %
|
|
tmp_dir, stdout, flags=re.M)
|
|
return match.group(1)
|
|
|
|
|
|
def check_sdist_contains_expected(sdist_dir, to_ship_files):
|
|
"""Check that the files we expect to ship are present in the sdist"""
|
|
results = []
|
|
for filename in to_ship_files:
|
|
path = os.path.join(sdist_dir, filename)
|
|
if not os.path.exists(path):
|
|
results.append('%s: File was not added to sdist' % filename)
|
|
|
|
# Also changelog
|
|
changelog_files = glob.glob(os.path.join(sdist_dir, 'changelogs/CHANGELOG-v2.[0-9]*.rst'))
|
|
if not changelog_files:
|
|
results.append('changelogs/CHANGELOG-v2.*.rst: Changelog file was not added to the sdist')
|
|
elif len(changelog_files) > 1:
|
|
results.append('changelogs/CHANGELOG-v2.*.rst: Too many changelog files: %s'
|
|
% changelog_files)
|
|
|
|
return results
|
|
|
|
|
|
def check_sdist_files_are_wanted(sdist_dir, to_ship_files):
|
|
"""Check that all files in the sdist are desired"""
|
|
results = []
|
|
for dirname, dummy, files in os.walk(sdist_dir):
|
|
dirname = os.path.relpath(dirname, start=sdist_dir)
|
|
if dirname == '.':
|
|
dirname = ''
|
|
|
|
for filename in files:
|
|
path = os.path.join(dirname, filename)
|
|
if path not in to_ship_files:
|
|
if fnmatch.fnmatch(path, 'changelogs/CHANGELOG-v2.[0-9]*.rst'):
|
|
# changelog files are expected
|
|
continue
|
|
|
|
# FIXME: ansible-test doesn't pass the paths of symlinks to us so we aren't
|
|
# checking those
|
|
if not os.path.islink(os.path.join(sdist_dir, path)):
|
|
results.append('%s: File in sdist was not in the repository' % path)
|
|
|
|
return results
|
|
|
|
|
|
def check_installed_contains_expected(install_dir, to_install_files):
|
|
"""Check that all the files we expect to be installed are"""
|
|
results = []
|
|
for filename in to_install_files:
|
|
path = os.path.join(install_dir, filename)
|
|
if not os.path.exists(path):
|
|
results.append('%s: File not installed' % os.path.join('lib', filename))
|
|
|
|
return results
|
|
|
|
|
|
EGG_RE = re.compile('ansible[^/]+\\.egg-info/(PKG-INFO|SOURCES.txt|'
|
|
'dependency_links.txt|not-zip-safe|requires.txt|top_level.txt)$')
|
|
|
|
|
|
def check_installed_files_are_wanted(install_dir, to_install_files):
|
|
"""Check that all installed files were desired"""
|
|
results = []
|
|
|
|
for dirname, dummy, files in os.walk(install_dir):
|
|
dirname = os.path.relpath(dirname, start=install_dir)
|
|
if dirname == '.':
|
|
dirname = ''
|
|
|
|
for filename in files:
|
|
# If this is a byte code cache, look for the python file's name
|
|
directory = dirname
|
|
if filename.endswith('.pyc') or filename.endswith('.pyo'):
|
|
# Remove the trailing "o" or c"
|
|
filename = filename[:-1]
|
|
|
|
if directory.endswith('%s__pycache__' % os.path.sep):
|
|
# Python3 byte code cache, look for the basename of
|
|
# __pycache__/__init__.cpython-36.py
|
|
segments = filename.rsplit('.', 2)
|
|
if len(segments) >= 3:
|
|
filename = '.'.join((segments[0], segments[2]))
|
|
directory = os.path.dirname(directory)
|
|
|
|
path = os.path.join(directory, filename)
|
|
|
|
# Test that the file was listed for installation
|
|
if path not in to_install_files:
|
|
# FIXME: ansible-test doesn't pass the paths of symlinks to us so we
|
|
# aren't checking those
|
|
if not os.path.islink(os.path.join(install_dir, path)):
|
|
if not EGG_RE.match(path):
|
|
results.append('%s: File was installed but was not supposed to be' % path)
|
|
|
|
return results
|
|
|
|
|
|
def _find_symlinks():
|
|
symlink_list = []
|
|
for dirname, directories, filenames in os.walk('.'):
|
|
for filename in filenames:
|
|
path = os.path.join(dirname, filename)
|
|
# Strip off "./" from the front
|
|
path = path[2:]
|
|
if os.path.islink(path):
|
|
symlink_list.append(path)
|
|
|
|
return symlink_list
|
|
|
|
|
|
def main():
|
|
"""All of the files in the repository"""
|
|
complete_file_list = []
|
|
for path in sys.argv[1:] or sys.stdin.read().splitlines():
|
|
complete_file_list.append(path)
|
|
|
|
# ansible-test isn't currently passing symlinks to us so construct those ourselves for now
|
|
for filename in _find_symlinks():
|
|
if filename not in complete_file_list:
|
|
# For some reason ansible-test is passing us lib/ansible/module_utils/ansible_release.py
|
|
# which is a symlink even though it doesn't pass any others
|
|
complete_file_list.append(filename)
|
|
|
|
# We may run this after docs sanity tests so get a clean repository to run in
|
|
with clean_repository(complete_file_list) as clean_repo_dir:
|
|
os.chdir(clean_repo_dir)
|
|
|
|
to_ship_files = assemble_files_to_ship(complete_file_list)
|
|
to_install_files = assemble_files_to_install(complete_file_list)
|
|
|
|
results = []
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
sdist_path = create_sdist(tmp_dir)
|
|
sdist_dir = extract_sdist(sdist_path, tmp_dir)
|
|
|
|
# Check that the files that are supposed to be in the sdist are there
|
|
results.extend(check_sdist_contains_expected(sdist_dir, to_ship_files))
|
|
|
|
# Check that the files that are in the sdist are in the repository
|
|
results.extend(check_sdist_files_are_wanted(sdist_dir, to_ship_files))
|
|
|
|
# install the sdist
|
|
install_dir = install_sdist(tmp_dir, sdist_dir)
|
|
|
|
# Check that the files that are supposed to be installed are there
|
|
results.extend(check_installed_contains_expected(install_dir, to_install_files))
|
|
|
|
# Check that the files that are installed are supposed to be installed
|
|
results.extend(check_installed_files_are_wanted(install_dir, to_install_files))
|
|
|
|
for message in results:
|
|
print(message)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|