cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

maven_artifact.py - add support for version ranges by using spec (#54309)

Browse source
This commit is contained in:
Pedro Magalhães 2019-08-21 14:33:28 +01:00 committed by John R Barker
parent 96199be96b
commit 145a6a8756
4 changed files with 181 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

123
lib/ansible/modules/packaging/language/maven_artifact.py
View file

@ -39,7 +39,14 @@ options:
version:
description:
- The maven version coordinate
default: latest
- Mutually exclusive with I(version_by_spec).
version_by_spec:
description:
- The maven dependency version ranges.
- See supported version ranges on U(https://cwiki.apache.org/confluence/display/MAVENOLD/Dependency+Mediation+and+Conflict+Resolution)
- The range type "(,1.0],[1.2,)" and "(,1.1),(1.1,)" is not supported.
- Mutually exclusive with I(version).
version_added: "2.9"
classifier:
description:
- The maven classifier coordinate
@ -90,7 +97,8 @@ options:
keep_name:
description:
- If C(yes), the downloaded artifact's name is preserved, i.e the version number remains part of it.
- This option only has effect when C(dest) is a directory and C(version) is set to C(latest).
- This option only has effect when C(dest) is a directory and C(version) is set to C(latest) or C(version_by_spec)
is defined.
type: bool
default: 'no'
version_added: "2.4"
@ -158,6 +166,13 @@ EXAMPLES = '''
artifact_id: junit
dest: /tmp/junit-latest.jar
repository_url: "file://{{ lookup('env','HOME') }}/.m2/repository"
# Download the latest version between 3.8 and 4.0 (exclusive) of the JUnit framework artifact from Maven Central
- maven_artifact:
group_id: junit
artifact_id: junit
version_by_spec: "[3.8,4.0)"
dest: /tmp/
'''
import hashlib
@ -168,6 +183,9 @@ import io
import tempfile
import traceback
from ansible.module_utils.ansible_release import __version__ as ansible_version
from re import match
LXML_ETREE_IMP_ERR = None
try:
from lxml import etree
@ -184,6 +202,15 @@ except ImportError:
BOTO_IMP_ERR = traceback.format_exc()
HAS_BOTO = False
SEMANTIC_VERSION_IMP_ERR = None
try:
from semantic_version import Version, Spec
HAS_SEMANTIC_VERSION = True
except ImportError:
SEMANTIC_VERSION_IMP_ERR = traceback.format_exc()
HAS_SEMANTIC_VERSION = False
from ansible.module_utils.basic import AnsibleModule, missing_required_lib
from ansible.module_utils.six.moves.urllib.parse import urlparse
from ansible.module_utils.urls import fetch_url
@ -224,7 +251,7 @@ def adjust_recursive_directory_permissions(pre_existing_dir, new_directory_list,
class Artifact(object):
def __init__(self, group_id, artifact_id, version, classifier='', extension='jar'):
def __init__(self, group_id, artifact_id, version, version_by_spec, classifier='', extension='jar'):
if not group_id:
raise ValueError("group_id must be set")
if not artifact_id:
@ -233,6 +260,7 @@ class Artifact(object):
self.group_id = group_id
self.artifact_id = artifact_id
self.version = version
self.version_by_spec = version_by_spec
self.classifier = classifier
if not extension:
@ -290,17 +318,62 @@ class Artifact(object):
class MavenDownloader:
def __init__(self, module, base="http://repo1.maven.org/maven2", local=False, headers=None):
def __init__(self, module, base, local=False, headers=None):
self.module = module
if base.endswith("/"):
base = base.rstrip("/")
self.base = base
self.local = local
self.headers = headers
self.user_agent = "Ansible {0} maven_artifact".format(self.module.ansible_version)
self.user_agent = "Ansible {0} maven_artifact".format(ansible_version)
self.latest_version_found = None
self.metadata_file_name = "maven-metadata-local.xml" if local else "maven-metadata.xml"
def find_version_by_spec(self, artifact):
path = "/%s/%s" % (artifact.path(False), self.metadata_file_name)
content = self._getContent(self.base + path, "Failed to retrieve the maven metadata file: " + path)
xml = etree.fromstring(content)
original_versions = xml.xpath("/metadata/versioning/versions/version/text()")
versions = []
for version in original_versions:
try:
versions.append(Version.coerce(version))
except ValueError:
# This means that version string is not a valid semantic versioning
pass
parse_versions_syntax = {
# example -> (,1.0]
r"^\(,(?P<upper_bound>[0-9.]*)]$": "<={upper_bound}",
# example -> 1.0
r"^(?P<version>[0-9.]*)$": "~={version}",
# example -> [1.0]
r"^\[(?P<version>[0-9.]*)\]$": "=={version}",
# example -> [1.2, 1.3]
r"^\[(?P<lower_bound>[0-9.]*),\s*(?P<upper_bound>[0-9.]*)\]$": ">={lower_bound},<={upper_bound}",
# example -> [1.2, 1.3)
r"^\[(?P<lower_bound>[0-9.]*),\s*(?P<upper_bound>[0-9.]+)\)$": ">={lower_bound},<{upper_bound}",
# example -> [1.5,)
r"^\[(?P<lower_bound>[0-9.]*),\)$": ">={lower_bound}",
}
for regex, spec_format in parse_versions_syntax.items():
regex_result = match(regex, artifact.version_by_spec)
if regex_result:
spec = Spec(spec_format.format(**regex_result.groupdict()))
selected_version = spec.select(versions)
if not selected_version:
raise ValueError("No version found with this spec version: {0}".format(artifact.version_by_spec))
# To deal when repos on maven don't have patch number on first build (e.g. 3.8 instead of 3.8.0)
if str(selected_version) not in original_versions:
selected_version.patch = None
return str(selected_version)
raise ValueError("The spec version {0} is not supported! ".format(artifact.version_by_spec))
def find_latest_version_available(self, artifact):
if self.latest_version_found:
return self.latest_version_found
@ -313,6 +386,9 @@ class MavenDownloader:
return v[0]
def find_uri_for_artifact(self, artifact):
if artifact.version_by_spec:
artifact.version = self.find_version_by_spec(artifact)
if artifact.version == "latest":
artifact.version = self.find_latest_version_available(artifact)
@ -390,8 +466,8 @@ class MavenDownloader:
return None
def download(self, tmpdir, artifact, verify_download, filename=None):
if not artifact.version or artifact.version == "latest":
artifact = Artifact(artifact.group_id, artifact.artifact_id, self.find_latest_version_available(artifact),
if (not artifact.version and not artifact.version_by_spec) or artifact.version == "latest":
artifact = Artifact(artifact.group_id, artifact.artifact_id, self.find_latest_version_available(artifact), None,
artifact.classifier, artifact.extension)
url = self.find_uri_for_artifact(artifact)
tempfd, tempname = tempfile.mkstemp(dir=tmpdir)
@ -464,10 +540,11 @@ def main():
argument_spec=dict(
group_id=dict(required=True),
artifact_id=dict(required=True),
version=dict(default="latest"),
version=dict(default=None),
version_by_spec=dict(default=None),
classifier=dict(default=''),
extension=dict(default='jar'),
repository_url=dict(default=None),
repository_url=dict(default='http://repo1.maven.org/maven2'),
username=dict(default=None, aliases=['aws_secret_key']),
password=dict(default=None, no_log=True, aliases=['aws_secret_access_key']),
headers=dict(type='dict'),
@ -478,12 +555,16 @@ def main():
keep_name=dict(required=False, default=False, type='bool'),
verify_checksum=dict(required=False, default='download', choices=['never', 'download', 'change', 'always'])
),
add_file_common_args=True
add_file_common_args=True,
mutually_exclusive=([('version', 'version_by_spec')])
)
if not HAS_LXML_ETREE:
module.fail_json(msg=missing_required_lib('lxml'), exception=LXML_ETREE_IMP_ERR)
if module.params['version_by_spec'] and not HAS_SEMANTIC_VERSION:
module.fail_json(msg=missing_required_lib('semantic_version'), exception=SEMANTIC_VERSION_IMP_ERR)
repository_url = module.params["repository_url"]
if not repository_url:
repository_url = "http://repo1.maven.org/maven2"
@ -501,6 +582,7 @@ def main():
group_id = module.params["group_id"]
artifact_id = module.params["artifact_id"]
version = module.params["version"]
version_by_spec = module.params["version_by_spec"]
classifier = module.params["classifier"]
extension = module.params["extension"]
headers = module.params['headers']
@ -514,8 +596,11 @@ def main():
downloader = MavenDownloader(module, repository_url, local, headers)
if not version_by_spec and not version:
version = "latest"
try:
artifact = Artifact(group_id, artifact_id, version, classifier, extension)
artifact = Artifact(group_id, artifact_id, version, version_by_spec, classifier, extension)
except ValueError as e:
module.fail_json(msg=e.args[0])
@ -537,13 +622,19 @@ def main():
if os.path.isdir(b_dest):
version_part = version
if keep_name and version == 'latest':
if version == 'latest':
version_part = downloader.find_latest_version_available(artifact)
elif version_by_spec:
version_part = downloader.find_version_by_spec(artifact)
filename = "{artifact_id}{version_part}{classifier}.{extension}".format(
artifact_id=artifact_id,
version_part="-{0}".format(version_part) if keep_name else "",
classifier="-{0}".format(classifier) if classifier else "",
extension=extension
)
dest = posixpath.join(dest, filename)
if classifier:
dest = posixpath.join(dest, "%s-%s-%s.%s" % (artifact_id, version_part, classifier, extension))
else:
dest = posixpath.join(dest, "%s-%s.%s" % (artifact_id, version_part, extension))
b_dest = to_bytes(dest, errors='surrogate_or_strict')
if os.path.lexists(b_dest) and ((not verify_change) or not downloader.is_invalid_md5(dest, downloader.find_uri_for_artifact(artifact))):

3
test/lib/ansible_test/_data/requirements/units.txt
View file

@ -47,3 +47,6 @@ httmock
# requirment for kubevirt modules
openshift ; python_version >= '2.7'
# requirement for maven_artifact
semantic_version

1
test/sanity/ignore.txt
View file

@ -5206,7 +5206,6 @@ lib/ansible/modules/packaging/language/easy_install.py validate-modules:E324
lib/ansible/modules/packaging/language/easy_install.py validate-modules:E337
lib/ansible/modules/packaging/language/easy_install.py validate-modules:E338
lib/ansible/modules/packaging/language/gem.py validate-modules:E337
lib/ansible/modules/packaging/language/maven_artifact.py validate-modules:E324
lib/ansible/modules/packaging/language/maven_artifact.py validate-modules:E337
lib/ansible/modules/packaging/language/maven_artifact.py validate-modules:E338
lib/ansible/modules/packaging/language/pear.py validate-modules:E322

71
test/units/modules/packaging/language/test_maven_artifact.py Normal file
View file

@ -0,0 +1,71 @@
# Copyright (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import pytest
from ansible.modules.packaging.language import maven_artifact
from ansible.module_utils import basic
pytestmark = pytest.mark.usefixtures('patch_ansible_module')
maven_metadata_example = b"""<?xml version="1.0" encoding="UTF-8"?>
<metadata>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<versioning>
<latest>4.13-beta-2</latest>
<release>4.13-beta-2</release>
<versions>
<version>3.7</version>
<version>3.8</version>
<version>3.8.1</version>
<version>3.8.2</version>
<version>4.0</version>
<version>4.1</version>
<version>4.2</version>
<version>4.3</version>
<version>4.3.1</version>
<version>4.4</version>
<version>4.5</version>
<version>4.6</version>
<version>4.7</version>
<version>4.8</version>
<version>4.8.1</version>
<version>4.8.2</version>
<version>4.9</version>
<version>4.10</version>
<version>4.11-beta-1</version>
<version>4.11</version>
<version>4.12-beta-1</version>
<version>4.12-beta-2</version>
<version>4.12-beta-3</version>
<version>4.12</version>
<version>4.13-beta-1</version>
<version>4.13-beta-2</version>
</versions>
<lastUpdated>20190202141051</lastUpdated>
</versioning>
</metadata>
"""
@pytest.mark.parametrize('patch_ansible_module, version_by_spec, version_choosed', [
(None, "(,3.9]", "3.8.2"),
(None, "3.0", "3.8.2"),
(None, "[3.7]", "3.7"),
(None, "[3.8]", "3.8.2"),
(None, "[4.10, 4.12]", "4.12"),
(None, "[4.10, 4.12)", "4.11"),
(None, "[2.0,)", "4.13-beta-2"),
])
def test_find_version_by_spec(mocker, version_by_spec, version_choosed):
_getContent = mocker.patch('ansible.modules.packaging.language.maven_artifact.MavenDownloader._getContent')
_getContent.return_value = maven_metadata_example
artifact = maven_artifact.Artifact("junit", "junit", None, version_by_spec, "jar")
mvn_downloader = maven_artifact.MavenDownloader(basic.AnsibleModule, "http://repo1.maven.org/maven2")
assert mvn_downloader.find_version_by_spec(artifact) == version_choosed