[win_get_url] feature: Add support checksum to module win_get_url (#51986)
* set valid_until equal to current time + spot_wait_timeout * Add checksum check for downloaded file. * refactoring * fix typo * add fixes * mart try,catch handling * revert lib/ansible/modules/cloud/amazon/ec2.py from upstream * refactoring * remove empty lines * add checksum verification for existing file * fix current file check * refactoring destination file check * add handling exceptions * refactoring * Added download file hash data from url * fix string aligning * fix bug with uri * Added get hash from multy-string file * Added URI support for checksum file location * refactoing * Remove any non-alphanumeric characters for hash from url * fix discussions; add support for PS3 * refactoring * add size return value * checkout from upstream for lib/ansible/modules/cloud/amazon/ec2.py * add Ansible.ModuleUtils.Legacy support; refactoring * Copyright added * Checking files size before and after downloading added. * remove unused code * Corrected regexp for dotted slashed file name prefix in hash-file * hotfix typo error; add int tests * remove legacy module support; split checksum to checksum, checksum_algorithm, checksum_url * changed default hash algorithm * Fixed case for ContentLength = -1 * Old comment removed * fix typo * Remove file size check before downloading * add alias to ; fix tests * adjust tests; fix lint warnings from PSScritpAnalyzer * workaround for bug in win_chocolatey module on win2008 * remove win_get_url.ps1 from /test/sanity/pslint/ignore.txt * add checksum_algorithm as retuen value * first normalise before return Result * resolve discussions Signed-off-by: Viktor Utkin <viktor.utkin7@yandex.ru> * fix discussions fix http tests as discussed * fix last discussions * Reduce code duplication and add idempotency check * fix sanity issue and remove testing code * move back to using tmp file for checksum comparison
This commit is contained in:
Viktor Utkin
Jordan Borean
parent
aafc5538bc
commit
b2a7561a7f
8 changed files with 616 additions and 156 deletions
3
changelogs/fragments/win_get_url-checksum.yaml
Normal file
3
changelogs/fragments/win_get_url-checksum.yaml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
minor_changes:
|
||||
- win_get_url - Add the ``checksum`` option to verify the integrity of a downloaded file.
|
||||
- win_get_url - Add idempotency check if the remote file has the same contents as the dest file.
|
||||
|
|
@ -3,10 +3,13 @@
|
|||
# Copyright: (c) 2015, Paul Durivage <paul.durivage@rackspace.com>
|
||||
# Copyright: (c) 2015, Tal Auslander <tal@cloudshare.com>
|
||||
# Copyright: (c) 2017, Dag Wieers <dag@wieers.com>
|
||||
# Copyright: (c) 2019, Viktor Utkin <viktor_utkin@epam.com>
|
||||
# Copyright: (c) 2019, Uladzimir Klybik <uladzimir_klybik@epam.com>
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
#AnsibleRequires -CSharpUtil Ansible.Basic
|
||||
#Requires -Module Ansible.ModuleUtils.AddType
|
||||
#Requires -Module Ansible.ModuleUtils.FileUtil
|
||||
|
||||
$spec = @{
|
||||
options = @{
|
||||
|
|
@ -23,7 +26,13 @@ $spec = @{
|
|||
proxy_username = @{ type='str' }
|
||||
proxy_password = @{ type='str'; no_log=$true }
|
||||
force = @{ type='bool'; default=$true }
|
||||
checksum = @{ type='str' }
|
||||
checksum_algorithm = @{ type='str'; default='sha1'; choices = @("md5", "sha1", "sha256", "sha384", "sha512") }
|
||||
checksum_url = @{ type='str' }
|
||||
}
|
||||
mutually_exclusive = @(
|
||||
,@('checksum', 'checksum_url')
|
||||
)
|
||||
supports_check_mode = $true
|
||||
}
|
||||
|
||||
|
|
@ -42,145 +51,271 @@ $proxy_url = $module.Params.proxy_url
|
|||
$proxy_username = $module.Params.proxy_username
|
||||
$proxy_password = $module.Params.proxy_password
|
||||
$force = $module.Params.force
|
||||
$checksum = $module.Params.checksum
|
||||
$checksum_algorithm = $module.Params.checksum_algorithm
|
||||
$checksum_url = $module.Params.checksum_url
|
||||
|
||||
Add-CSharpType -AnsibleModule $module -References @'
|
||||
using System.Net;
|
||||
public class ExtendedWebClient : WebClient {
|
||||
public int Timeout;
|
||||
|
||||
public ExtendedWebClient() {
|
||||
Timeout = 600000; // Default timeout value
|
||||
}
|
||||
|
||||
protected override WebRequest GetWebRequest(System.Uri address) {
|
||||
WebRequest request = base.GetWebRequest(address);
|
||||
request.Timeout = Timeout;
|
||||
return request;
|
||||
}
|
||||
}
|
||||
'@
|
||||
|
||||
|
||||
Function CheckModified-File($module, $url, $dest, $headers, $credentials, $timeout, $use_proxy, $proxy) {
|
||||
|
||||
$fileLastMod = ([System.IO.FileInfo]$dest).LastWriteTimeUtc
|
||||
$webLastMod = $null
|
||||
|
||||
$webRequest = [System.Net.WebRequest]::Create($url)
|
||||
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
$webRequest.Headers.Add($header.Name, $header.Value)
|
||||
}
|
||||
|
||||
if ($timeout) {
|
||||
$webRequest.Timeout = $timeout * 1000
|
||||
}
|
||||
|
||||
if (-not $use_proxy) {
|
||||
# Ignore the system proxy settings
|
||||
$webRequest.Proxy = $null
|
||||
} elseif ($proxy) {
|
||||
$webRequest.Proxy = $proxy
|
||||
}
|
||||
|
||||
if ($credentials) {
|
||||
if ($force_basic_auth) {
|
||||
$webRequest.Headers.Add("Authorization", "Basic $credentials")
|
||||
} else {
|
||||
$webRequest.Credentials = $credentials
|
||||
}
|
||||
}
|
||||
|
||||
if ($webRequest -is [System.Net.FtpWebRequest]) {
|
||||
$webRequest.Method = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp
|
||||
} else {
|
||||
$webRequest.Method = [System.Net.WebRequestMethods+Http]::Head
|
||||
}
|
||||
|
||||
# FIXME: Split both try-statements and single-out catched exceptions with more specific error messages
|
||||
Try {
|
||||
$webResponse = $webRequest.GetResponse()
|
||||
$webLastMod = $webResponse.LastModified
|
||||
} Catch [System.Net.WebException] {
|
||||
$module.Result.status_code = [int] $_.Exception.Response.StatusCode
|
||||
$module.FailJson("Error requesting '$url'. $($_.Exception.Message)", $_)
|
||||
} Catch {
|
||||
$module.FailJson("Error when requesting 'Last-Modified' date from '$url'. $($_.Exception.Message)", $_)
|
||||
}
|
||||
$module.Result.status_code = [int] $webResponse.StatusCode
|
||||
$module.Result.msg = [string] $webResponse.StatusDescription
|
||||
$webResponse.Close()
|
||||
|
||||
if ($webLastMod -and ((Get-Date -Date $webLastMod).ToUniversalTime() -lt $fileLastMod)) {
|
||||
return $false
|
||||
} else {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Function Download-File($module, $url, $dest, $headers, $credentials, $timeout, $use_proxy, $proxy) {
|
||||
|
||||
$module_start = Get-Date
|
||||
|
||||
# Check $dest parent folder exists before attempting download, which avoids unhelpful generic error message.
|
||||
$dest_parent = Split-Path -LiteralPath $dest
|
||||
if (-not (Test-Path -LiteralPath $dest_parent -PathType Container)) {
|
||||
$module.FailJson("The path '$dest_parent' does not exist for destination '$dest', or is not visible to the current user. Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.")
|
||||
}
|
||||
|
||||
# TODO: Replace this with WebRequest
|
||||
$extWebClient = New-Object ExtendedWebClient
|
||||
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
$extWebClient.Headers.Add($header.Name, $header.Value)
|
||||
}
|
||||
|
||||
if ($timeout) {
|
||||
$extWebClient.Timeout = $timeout * 1000
|
||||
}
|
||||
|
||||
if (-not $use_proxy) {
|
||||
# Ignore the system proxy settings
|
||||
$extWebClient.Proxy = $null
|
||||
} elseif ($proxy) {
|
||||
$extWebClient.Proxy = $proxy
|
||||
}
|
||||
|
||||
if ($credentials) {
|
||||
if ($force_basic_auth) {
|
||||
$extWebClient.Headers.Add("Authorization","Basic $credentials")
|
||||
} else {
|
||||
$extWebClient.Credentials = $credentials
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $module.CheckMode) {
|
||||
# FIXME: Single-out catched exceptions with more specific error messages
|
||||
Try {
|
||||
$extWebClient.DownloadFile($url, $dest)
|
||||
} Catch [System.Net.WebException] {
|
||||
$module.Result.status_code = [int] $_.Exception.Response.StatusCode
|
||||
$module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
|
||||
$module.FailJson("Error downloading '$url' to '$dest': $($_.Exception.Message)", $_)
|
||||
} Catch {
|
||||
$module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
|
||||
$module.FailJson("Unknown error downloading '$url' to '$dest': $($_.Exception.Message)", $_)
|
||||
}
|
||||
}
|
||||
|
||||
$module.Result.status_code = 200
|
||||
$module.Result.changed = $true
|
||||
$module.Result.msg = 'OK'
|
||||
$module.Result.dest = $dest
|
||||
$module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
|
||||
|
||||
}
|
||||
|
||||
$module.Result.dest = $dest
|
||||
$module.Result.elapsed = 0
|
||||
$module.Result.url = $url
|
||||
|
||||
Function Invoke-AnsibleWebRequest {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Creates a WebRequest and invokes a ScriptBlock with the response passed in.
|
||||
It handles the common module options like credential, timeout, proxy options
|
||||
in a single location to reduce code duplication.
|
||||
#>
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
|
||||
[Parameter(Mandatory=$true)][Uri]$Uri,
|
||||
[Parameter(Mandatory=$true)][Hashtable]$Method,
|
||||
[Parameter(Mandatory=$true)][ScriptBlock]$Script, # Invoked in this cmdlet
|
||||
[System.Collections.IDictionary]$Headers,
|
||||
[Int32]$Timeout,
|
||||
[Switch]$UseProxy,
|
||||
[System.Net.WebProxy]$Proxy,
|
||||
$Credential # Either a String (force_basic_auth) or NetCredentials
|
||||
)
|
||||
|
||||
$web_request = [System.Net.WebRequest]::Create($Uri)
|
||||
$web_request.Method = $Method.($web_request.GetType().Name)
|
||||
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
$web_request.Headers.Add($header.Key, $header.Value)
|
||||
}
|
||||
|
||||
if ($timeout) {
|
||||
$web_request.Timeout = $timeout * 1000
|
||||
}
|
||||
|
||||
if (-not $UseProxy) {
|
||||
$web_request.Proxy = $null
|
||||
} elseif ($ProxyUri) {
|
||||
$web_request.Proxy = $Proxy
|
||||
}
|
||||
|
||||
if ($Credential) {
|
||||
if ($Credential -is [String]) {
|
||||
# force_basic_auth=yes is set
|
||||
$web_request.Headers.Add("Authorization", "Basic $Credential")
|
||||
} else {
|
||||
$web_request.Credentials = $Credential
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
$web_response = $web_request.GetResponse()
|
||||
$response_stream = $web_response.GetResponseStream()
|
||||
try {
|
||||
# Invoke the ScriptBlock and pass in the WebResponse and ResponseStream
|
||||
&$Script -Response $web_response -Stream $response_stream
|
||||
} finally {
|
||||
$response_stream.Dispose()
|
||||
}
|
||||
|
||||
if ($Uri.IsFile) {
|
||||
# A FileWebResponse won't have these properties set
|
||||
$module.Result.msg = "OK"
|
||||
$module.Result.status_code = 200
|
||||
} else {
|
||||
$module.Result.msg = [string]$web_response.StatusDescription
|
||||
$module.Result.status_code = [int]$web_response.StatusCode
|
||||
}
|
||||
} catch [System.Net.WebException] {
|
||||
$module.Result.status_code = [int]$_.Exception.Response.StatusCode
|
||||
$module.FailJson("Error requesting '$Uri'. $($_.Exception.Message)", $_)
|
||||
} finally {
|
||||
if ($web_response) {
|
||||
$web_response.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Function Get-ChecksumFromUri {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
|
||||
[Parameter(Mandatory=$true)][Uri]$Uri,
|
||||
[Uri]$SourceUri,
|
||||
[Hashtable]$RequestParams
|
||||
)
|
||||
|
||||
$script = {
|
||||
param($Response, $Stream)
|
||||
|
||||
$read_stream = New-Object -TypeName System.IO.StreamReader -ArgumentList $Stream
|
||||
$web_checksum = $read_stream.ReadToEnd()
|
||||
$basename = (Split-Path -Path $SourceUri.LocalPath -Leaf)
|
||||
$basename = [regex]::Escape($basename)
|
||||
$web_checksum_str = $web_checksum -split '\r?\n' | Select-String -Pattern $("\s+\.?\/?\\?" + $basename + "\s*$")
|
||||
if (-not $web_checksum_str) {
|
||||
$Module.FailJson("Checksum record not found for file name '$basename' in file from url: '$Uri'")
|
||||
}
|
||||
|
||||
$web_checksum_str_splitted = $web_checksum_str[0].ToString().split(" ", 2)
|
||||
$hash_from_file = $web_checksum_str_splitted[0].Trim()
|
||||
# Remove any non-alphanumeric characters
|
||||
$hash_from_file = $hash_from_file -replace '\W+', ''
|
||||
|
||||
Write-Output -InputObject $hash_from_file
|
||||
}
|
||||
$invoke_args = @{
|
||||
Module = $Module
|
||||
Uri = $Uri
|
||||
Method = @{
|
||||
FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile
|
||||
FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile
|
||||
HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get
|
||||
}
|
||||
Script = $script
|
||||
}
|
||||
try {
|
||||
Invoke-AnsibleWebRequest @invoke_args @RequestParams
|
||||
} catch {
|
||||
$Module.FailJson("Error when getting the remote checksum from '$Uri'. $($_.Exception.Message)", $_)
|
||||
}
|
||||
}
|
||||
|
||||
Function Compare-ModifiedFile {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Compares the remote URI resource against the local Dest resource. Will
|
||||
return true if the LastWriteTime/LastModificationDate of the remote is
|
||||
newer than the local resource date.
|
||||
#>
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
|
||||
[Parameter(Mandatory=$true)][Uri]$Uri,
|
||||
[Parameter(Mandatory=$true)][String]$Dest,
|
||||
[Hashtable]$RequestParams
|
||||
)
|
||||
|
||||
$dest_last_mod = (Get-AnsibleItem -Path $Dest).LastWriteTimeUtc
|
||||
|
||||
# If the URI is a file we don't need to go through the whole WebRequest
|
||||
if ($Uri.IsFile) {
|
||||
$src_last_mod = (Get-AnsibleItem -Path $Uri.AbsolutePath).LastWriteTimeUtc
|
||||
} else {
|
||||
$invoke_args = @{
|
||||
Module = $Module
|
||||
Uri = $Uri
|
||||
Method = @{
|
||||
FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp
|
||||
HttpWebRequest = [System.Net.WebRequestMethods+Http]::Head
|
||||
}
|
||||
Script = { param($Response, $Stream); $Response.LastModified }
|
||||
}
|
||||
try {
|
||||
$src_last_mod = Invoke-AnsibleWebRequest @invoke_args @RequestParams
|
||||
} catch {
|
||||
$Module.FailJson("Error when requesting 'Last-Modified' date from '$Uri'. $($_.Exception.Message)", $_)
|
||||
}
|
||||
}
|
||||
|
||||
# Return $true if the Uri LastModification date is newer than the Dest LastModification date
|
||||
((Get-Date -Date $src_last_mod).ToUniversalTime() -gt $dest_last_mod)
|
||||
}
|
||||
|
||||
Function Get-Checksum {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][String]$Path,
|
||||
[String]$Algorithm = "sha1"
|
||||
)
|
||||
|
||||
switch ($Algorithm) {
|
||||
'md5' { $sp = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider }
|
||||
'sha1' { $sp = New-Object -TypeName System.Security.Cryptography.SHA1CryptoServiceProvider }
|
||||
'sha256' { $sp = New-Object -TypeName System.Security.Cryptography.SHA256CryptoServiceProvider }
|
||||
'sha384' { $sp = New-Object -TypeName System.Security.Cryptography.SHA384CryptoServiceProvider }
|
||||
'sha512' { $sp = New-Object -TypeName System.Security.Cryptography.SHA512CryptoServiceProvider }
|
||||
}
|
||||
|
||||
$fs = [System.IO.File]::Open($Path, [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read,
|
||||
[System.IO.FileShare]::ReadWrite)
|
||||
try {
|
||||
$hash = [System.BitConverter]::ToString($sp.ComputeHash($fs)).Replace("-", "").ToLower()
|
||||
} finally {
|
||||
$fs.Dispose()
|
||||
}
|
||||
return $hash
|
||||
}
|
||||
|
||||
Function Invoke-DownloadFile {
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
|
||||
[Parameter(Mandatory=$true)][Uri]$Uri,
|
||||
[Parameter(Mandatory=$true)][String]$Dest,
|
||||
[String]$Checksum,
|
||||
[String]$ChecksumAlgorithm,
|
||||
[Hashtable]$RequestParams
|
||||
)
|
||||
|
||||
# Check $dest parent folder exists before attempting download, which avoids unhelpful generic error message.
|
||||
$dest_parent = Split-Path -LiteralPath $Dest
|
||||
if (-not (Test-Path -LiteralPath $dest_parent -PathType Container)) {
|
||||
$module.FailJson("The path '$dest_parent' does not exist for destination '$Dest', or is not visible to the current user. Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.")
|
||||
}
|
||||
|
||||
$download_script = {
|
||||
param($Response, $Stream)
|
||||
|
||||
# Download the file to a temporary directory so we can compare it
|
||||
$tmp_dest = Join-Path -Path $Module.Tmpdir -ChildPath ([System.IO.Path]::GetRandomFileName())
|
||||
$fs = [System.IO.File]::Create($tmp_dest)
|
||||
try {
|
||||
$Stream.CopyTo($fs)
|
||||
$fs.Flush()
|
||||
} finally {
|
||||
$fs.Dispose()
|
||||
}
|
||||
$tmp_checksum = Get-Checksum -Path $tmp_dest -Algorithm $ChecksumAlgorithm
|
||||
$Module.Result.checksum_src = $tmp_checksum
|
||||
|
||||
# If the checksum has been set, verify the checksum of the remote against the input checksum.
|
||||
if ($Checksum -and $Checksum -ne $tmp_checksum) {
|
||||
$Module.FailJson(("The checksum for {0} did not match '{1}', it was '{2}'" -f $Uri, $Checksum, $tmp_checksum))
|
||||
}
|
||||
|
||||
$download = $true
|
||||
if (Test-Path -LiteralPath $Dest) {
|
||||
# Validate the remote checksum against the existing downloaded file
|
||||
$dest_checksum = Get-Checksum -Path $Dest -Algorithm $ChecksumAlgorithm
|
||||
|
||||
# If we don't need to download anything, save the dest checksum so we don't waste time calculating it
|
||||
# again at the end of the script
|
||||
if ($dest_checksum -eq $tmp_checksum) {
|
||||
$download = $false
|
||||
$Module.Result.checksum_dest = $dest_checksum
|
||||
$Module.Result.size = (Get-AnsibleItem -Path $Dest).Length
|
||||
}
|
||||
}
|
||||
|
||||
if ($download) {
|
||||
Copy-Item -Path $tmp_dest -Destination $Dest -Force -WhatIf:$Module.CheckMode > $null
|
||||
$Module.Result.changed = $true
|
||||
}
|
||||
}
|
||||
|
||||
$invoke_args = @{
|
||||
Module = $Module
|
||||
Uri = $Uri
|
||||
Method = @{
|
||||
FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile
|
||||
FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile
|
||||
HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get
|
||||
}
|
||||
Script = $download_script
|
||||
}
|
||||
|
||||
$module_start = Get-Date
|
||||
try {
|
||||
Invoke-AnsibleWebRequest @invoke_args @RequestParams
|
||||
} catch {
|
||||
$Module.FailJson("Unknown error downloading '$Uri' to '$Dest': $($_.Exception.Message)", $_)
|
||||
} finally {
|
||||
$Module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $use_proxy -and ($proxy_url -or $proxy_username -or $proxy_password)) {
|
||||
$module.Warn("Not using a proxy on request, however a 'proxy_url', 'proxy_username' or 'proxy_password' was defined.")
|
||||
}
|
||||
|
|
@ -224,6 +359,7 @@ if (Test-Path -LiteralPath $dest -PathType Container) {
|
|||
# We have a trailing path separator
|
||||
$module.FailJson("The destination path '$dest' does not exist, or is not visible to the current user. Ensure download destination folder exists (perhaps using win_file state=directory) before win_get_url runs.")
|
||||
}
|
||||
|
||||
$module.Result.dest = $dest
|
||||
|
||||
# Enable TLS1.1/TLS1.2 if they're available but disabled (eg. .NET 4.5)
|
||||
|
|
@ -236,22 +372,53 @@ if ([Net.SecurityProtocolType].GetMember("Tls12").Count -gt 0) {
|
|||
}
|
||||
[Net.ServicePointManager]::SecurityProtocol = $security_protocols
|
||||
|
||||
if ($force -or -not (Test-Path -LiteralPath $dest)) {
|
||||
$request_params = @{
|
||||
Credential = $credentials
|
||||
Headers = $headers
|
||||
Timeout = $timeout
|
||||
UseProxy = $use_proxy
|
||||
Proxy = $proxy
|
||||
}
|
||||
|
||||
Download-File -module $module -url $url -dest $dest -credentials $credentials `
|
||||
-headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy
|
||||
|
||||
} else {
|
||||
|
||||
$is_modified = CheckModified-File -module $module -url $url -dest $dest -credentials $credentials `
|
||||
-headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy
|
||||
|
||||
if ($is_modified) {
|
||||
|
||||
Download-File -module $module -url $url -dest $dest -credentials $credentials `
|
||||
-headers $headers -timeout $timeout -use_proxy $use_proxy -proxy $proxy
|
||||
if ($checksum) {
|
||||
$checksum = $checksum.Trim().toLower()
|
||||
}
|
||||
if ($checksum_algorithm) {
|
||||
$checksum_algorithm = $checksum_algorithm.Trim().toLower()
|
||||
}
|
||||
if ($checksum_url) {
|
||||
$checksum_url = $checksum_url.Trim().toLower()
|
||||
}
|
||||
|
||||
# Check for case $checksum variable contain url. If yes, get file data from url and replace original value in $checksum
|
||||
if ($checksum_url) {
|
||||
$checksum_uri = [System.Uri]$checksum_url
|
||||
if ($checksum_uri.Scheme -notin @("file", "ftp", "http", "https")) {
|
||||
$module.FailJson("Unsupported 'checksum_url' value for '$dest': '$checksum_url'")
|
||||
}
|
||||
|
||||
$checksum = Get-ChecksumFromUri -Module $Module -Uri $checksum_uri -SourceUri $url -RequestParams $request_params
|
||||
}
|
||||
|
||||
if ($force -or -not (Test-Path -LiteralPath $dest)) {
|
||||
# force=yes or dest does not exist, download the file
|
||||
# Note: Invoke-DownloadFile will compare the checksums internally if dest exists
|
||||
Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum `
|
||||
-ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params
|
||||
} else {
|
||||
# force=no, we want to check the last modified dates and only download if they don't match
|
||||
$is_modified = Compare-ModifiedFile -Module $module -Uri $url -Dest $dest -RequestParams $request_params
|
||||
if ($is_modified) {
|
||||
Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum `
|
||||
-ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params
|
||||
}
|
||||
}
|
||||
|
||||
if ((-not $module.Result.ContainsKey("checksum_dest")) -and (Test-Path -LiteralPath $dest)) {
|
||||
# Calculate the dest file checksum if it hasn't already been done
|
||||
$module.Result.checksum_dest = Get-Checksum -Path $dest -Algorithm $checksum_algorithm
|
||||
$module.Result.size = (Get-AnsibleItem -Path $dest).Length
|
||||
}
|
||||
|
||||
$module.ExitJson()
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ options:
|
|||
required: yes
|
||||
force:
|
||||
description:
|
||||
- If C(yes), will always download the file. If C(no), will only
|
||||
- If C(yes), will download the file every time and replace the file if the contents change. If C(no), will only
|
||||
download the file if it does not exist or the remote file has been
|
||||
modified more recently than the local file.
|
||||
- This works by sending an http HEAD request to retrieve last modified
|
||||
|
|
@ -73,6 +73,36 @@ options:
|
|||
type: bool
|
||||
default: yes
|
||||
version_added: '2.4'
|
||||
checksum:
|
||||
description:
|
||||
- If a I(checksum) is passed to this parameter, the digest of the
|
||||
destination file will be calculated after it is downloaded to ensure
|
||||
its integrity and verify that the transfer completed successfully.
|
||||
- This option cannot be set with I(checksum_url).
|
||||
type: str
|
||||
version_added: "2.8"
|
||||
checksum_algorithm:
|
||||
description:
|
||||
- Specifies the hashing algorithm used when calculating the checksum of
|
||||
the remote and destination file.
|
||||
type: str
|
||||
choices:
|
||||
- md5
|
||||
- sha1
|
||||
- sha256
|
||||
- sha385
|
||||
- sha512
|
||||
default: sha1
|
||||
version_added: "2.8"
|
||||
checksum_url:
|
||||
description:
|
||||
- Specifies a URL that contains the checksum values for the resource at
|
||||
I(url).
|
||||
- Like C(checksum), this is used to verify the integrity of the remote
|
||||
transfer.
|
||||
- This option cannot be set with I(checksum).
|
||||
type: str
|
||||
version_added: "2.8"
|
||||
proxy_url:
|
||||
description:
|
||||
- The full URL of the proxy server to download through.
|
||||
|
|
@ -105,6 +135,9 @@ notes:
|
|||
- If your URL includes an escaped slash character (%2F) this module will convert it to a real slash.
|
||||
This is a result of the behaviour of the System.Uri class as described in
|
||||
L(the documentation,https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/network/schemesettings-element-uri-settings#remarks).
|
||||
- Since Ansible 2.8, the module will skip reporting a change if the remote
|
||||
checksum is the same as the local local even when C(force=yes). This is to
|
||||
better align with M(get_url).
|
||||
seealso:
|
||||
- module: get_url
|
||||
- module: uri
|
||||
|
|
@ -140,6 +173,22 @@ EXAMPLES = r'''
|
|||
dest: '%TEMP%\ftp-file.txt'
|
||||
url_username: ftp-user
|
||||
url_password: ftp-password
|
||||
|
||||
- name: Download src with sha256 checksum url
|
||||
win_get_url:
|
||||
url: http://www.example.com/earthrise.jpg
|
||||
dest: C:\temp\earthrise.jpg
|
||||
checksum_url: http://www.example.com/sha256sum.txt
|
||||
checksum_algorithm: sha256
|
||||
force: True
|
||||
|
||||
- name: Download src with sha256 checksum url
|
||||
win_get_url:
|
||||
url: http://www.example.com/earthrise.jpg
|
||||
dest: C:\temp\earthrise.jpg
|
||||
checksum: a97e6837f60cec6da4491bab387296bbcd72bdba
|
||||
checksum_algorithm: sha1
|
||||
force: True
|
||||
'''
|
||||
|
||||
RETURN = r'''
|
||||
|
|
@ -148,11 +197,26 @@ dest:
|
|||
returned: always
|
||||
type: str
|
||||
sample: C:\Users\RandomUser\earthrise.jpg
|
||||
checksum_dest:
|
||||
description: <algorithm> checksum of the file after the download
|
||||
returned: success and dest has been downloaded
|
||||
type: str
|
||||
sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827
|
||||
checksum_src:
|
||||
description: <algorithm> checksum of the remote resource
|
||||
returned: force=yes or dest did not exist
|
||||
type: str
|
||||
sample: 6e642bb8dd5c2e027bf21dd923337cbb4214f827
|
||||
elapsed:
|
||||
description: The elapsed seconds between the start of poll and the end of the module.
|
||||
returned: always
|
||||
type: float
|
||||
sample: 2.1406487
|
||||
size:
|
||||
description: size of the dest file
|
||||
returned: success
|
||||
type: int
|
||||
sample: 1220
|
||||
url:
|
||||
description: requested url
|
||||
returned: always
|
||||
|
|
|
|||
|
|
@ -2,3 +2,6 @@
|
|||
test_win_get_url_path: '{{win_output_dir}}\win_get_url'
|
||||
test_win_get_url_host: www.redhat.com
|
||||
test_win_get_url_env_var: WIN_GET_URL
|
||||
|
||||
remote_tmp_path: '{{win_output_dir}}\win_get_url_dest'
|
||||
remote_http_path: '{{test_win_get_url_path}}\http'
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
---
|
||||
- name: ensure testing folder is present
|
||||
win_file:
|
||||
path: '{{test_win_get_url_path}}'
|
||||
path: '{{ item }}'
|
||||
state: directory
|
||||
loop:
|
||||
- '{{ test_win_get_url_path }}'
|
||||
- '{{ remote_http_path }}'
|
||||
- '{{ remote_tmp_path }}'
|
||||
|
||||
- name: copy across testing files
|
||||
win_copy:
|
||||
|
|
@ -41,6 +45,9 @@
|
|||
- name: run FTP tests
|
||||
include_tasks: tests_ftp.yml
|
||||
|
||||
- name: run checksum tests
|
||||
include_tasks: tests_checksum.yml
|
||||
|
||||
always:
|
||||
- name: remove SlimFTPd service
|
||||
win_service:
|
||||
|
|
@ -53,7 +60,10 @@
|
|||
level: machine
|
||||
state: absent
|
||||
|
||||
- name: remove testing folder
|
||||
- name: remove testing folders
|
||||
win_file:
|
||||
path: '{{test_win_get_url_path}}'
|
||||
path: '{{ item }}'
|
||||
state: absent
|
||||
loop:
|
||||
- '{{ test_win_get_url_path }}'
|
||||
- '{{ remote_tmp_path }}'
|
||||
|
|
|
|||
202
test/integration/targets/win_get_url/tasks/tests_checksum.yml
Normal file
202
test/integration/targets/win_get_url/tasks/tests_checksum.yml
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
---
|
||||
- name: create src file
|
||||
win_copy:
|
||||
dest: '{{ remote_http_path }}\27617.txt'
|
||||
content: 'ptux'
|
||||
|
||||
- name: create sha1 checksum file of src
|
||||
win_copy:
|
||||
dest: '{{ remote_http_path }}\sha1sum.txt'
|
||||
content: 'a97e6837f60cec6da4491bab387296bbcd72bdba 27617.txt'
|
||||
|
||||
- name: add sha1 checksum not going to be downloaded
|
||||
win_lineinfile:
|
||||
dest: '{{ remote_http_path }}\sha1sum.txt'
|
||||
line: '{{ item }}'
|
||||
loop:
|
||||
- '3911340502960ca33aece01129234460bfeb2791 not_target1.txt'
|
||||
- '1b4b6adf30992cedb0f6edefd6478ff0a593b2e4 not_target2.txt'
|
||||
|
||||
- name: create sha256 checksum file of src
|
||||
win_copy:
|
||||
dest: '{{ remote_http_path }}\sha256sum.txt'
|
||||
content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. 27617.txt'
|
||||
|
||||
- name: add sha256 checksum not going to be downloaded
|
||||
win_lineinfile:
|
||||
dest: '{{ remote_http_path }}\sha256sum.txt'
|
||||
line: '{{ item }}'
|
||||
loop:
|
||||
- '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 not_target1.txt'
|
||||
- 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b not_target2.txt'
|
||||
|
||||
- name: create sha256 checksum file of src with a dot leading path
|
||||
win_copy:
|
||||
dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
|
||||
content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. ./27617.txt'
|
||||
|
||||
- name: add sha256 checksums with dot leading path not going to be downloaded
|
||||
win_lineinfile:
|
||||
dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
|
||||
line: '{{ item }}'
|
||||
loop:
|
||||
- '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 ./not_target1.txt'
|
||||
- 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b ./not_target2.txt'
|
||||
|
||||
- name: copy files to ftp
|
||||
win_copy:
|
||||
src: '{{ remote_http_path }}\'
|
||||
dest: '{{ test_win_get_url_path }}\ftp\{{ item }}\'
|
||||
remote_src: yes
|
||||
loop: '{{ dest_folders }}'
|
||||
vars:
|
||||
dest_folders:
|
||||
- anon
|
||||
- user
|
||||
- user-pass
|
||||
|
||||
- name: download src with sha1 checksum url
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}'
|
||||
checksum_url: 'ftp://localhost/anon/sha1sum.txt'
|
||||
force: True
|
||||
register: result_sha1
|
||||
|
||||
- name: download src with sha1 checksum value
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}'
|
||||
checksum: 'a97e6837f60cec6da4491bab387296bbcd72bdba'
|
||||
force: True
|
||||
register: result_sha1_alt
|
||||
|
||||
- win_stat:
|
||||
path: '{{ remote_tmp_path }}\27617.txt'
|
||||
register: stat_result_sha1
|
||||
|
||||
- name: download src with sha256 checksum url
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}\27617sha256.txt'
|
||||
checksum_url: 'ftp://localhost/anon/sha256sum.txt'
|
||||
checksum_algorithm: sha256
|
||||
force: True
|
||||
register: result_sha256
|
||||
|
||||
- win_stat:
|
||||
path: '{{ remote_tmp_path }}\27617.txt'
|
||||
register: stat_result_sha256
|
||||
|
||||
- name: download src with sha256 checksum url with dot leading paths
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
|
||||
checksum_url: 'ftp://localhost/anon/sha256sum_with_dot.txt'
|
||||
checksum_algorithm: sha256
|
||||
force: True
|
||||
register: result_sha256_with_dot
|
||||
|
||||
- win_stat:
|
||||
path: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
|
||||
register: stat_result_sha256_with_dot
|
||||
|
||||
- name: Assert that the file was downloaded
|
||||
assert:
|
||||
that:
|
||||
- result_sha1 is changed
|
||||
- result_sha1.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
|
||||
- result_sha1_alt is succeeded
|
||||
- not result_sha1_alt is changed
|
||||
- result_sha1_alt.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
|
||||
- result_sha256 is changed
|
||||
- result_sha256_with_dot is changed
|
||||
- stat_result_sha1.stat.exists | bool
|
||||
- stat_result_sha256.stat.exists | bool
|
||||
- stat_result_sha256_with_dot.stat.exists | bool
|
||||
|
||||
# Check download with force: False
|
||||
|
||||
- name: download src with sha1 checksum url
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}'
|
||||
checksum_url: 'ftp://localhost/anon/sha1sum.txt'
|
||||
checksum_algorithm: sha1
|
||||
force: False
|
||||
register: result_sha1_no_force
|
||||
|
||||
- name: download src with sha256 checksum url
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}/27617sha256.txt'
|
||||
checksum_url: 'ftp://localhost/anon/sha256sum.txt'
|
||||
checksum_algorithm: sha256
|
||||
force: False
|
||||
register: result_sha256_no_force
|
||||
|
||||
- name: assert download single file with force no
|
||||
assert:
|
||||
that:
|
||||
- result_sha1_no_force is not changed
|
||||
- result_sha256_no_force is not changed
|
||||
|
||||
# Check download with check_mode: True
|
||||
|
||||
- name: download src with sha1 checksum url | checkmode
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}'
|
||||
checksum_url: 'ftp://localhost/anon/sha1sum.txt'
|
||||
checksum_algorithm: sha1
|
||||
check_mode: True
|
||||
register: result_sha1_check_mode
|
||||
|
||||
- name: download src with sha256 checksum url | checkmode
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}\27617sha256.txt'
|
||||
checksum_url: 'ftp://localhost/anon/sha256sum.txt'
|
||||
checksum_algorithm: sha256
|
||||
check_mode: True
|
||||
register: result_sha256_check_mode
|
||||
|
||||
- name: assert download single file with force no
|
||||
assert:
|
||||
that:
|
||||
- result_sha1_check_mode.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
|
||||
- result_sha256_check_mode.checksum_dest == 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006'
|
||||
- result_sha1_check_mode.checksum_src | length
|
||||
- result_sha256_check_mode.checksum_src | length
|
||||
|
||||
# Check download with wrong checksum
|
||||
|
||||
- name: download src with sha1 checksum value | fail
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}'
|
||||
checksum: 'redacted'
|
||||
checksum_algorithm: sha1
|
||||
failed_when:
|
||||
- '"did not match" not in result_sha1_failed.msg'
|
||||
- '"The checksum" not in result_sha1_failed.msg'
|
||||
- '"Unknown error downloading" not in result_sha1_failed.msg'
|
||||
register: result_sha1_failed
|
||||
|
||||
- name: download src with sha256 checksum value | fail
|
||||
win_get_url:
|
||||
url: 'ftp://localhost/anon/27617.txt'
|
||||
dest: '{{ remote_tmp_path }}\27617sha256.txt'
|
||||
checksum: 'redacted'
|
||||
checksum_algorithm: sha256
|
||||
failed_when:
|
||||
- '"did not match" not in result_sha256_failed.msg'
|
||||
- '"The checksum" not in result_sha256_failed.msg'
|
||||
- '"Unknown error downloading" not in result_sha256_failed.msg'
|
||||
register: result_sha256_failed
|
||||
|
||||
- name: assert failed downloads
|
||||
assert:
|
||||
that:
|
||||
- result_sha1_failed is succeeded
|
||||
- result_sha256_failed is succeeded
|
||||
|
|
@ -39,7 +39,16 @@
|
|||
- http_download.dest
|
||||
- http_download_result.stat.exists
|
||||
|
||||
# TODO: add check for idempotent run once it is added with force: yes
|
||||
- name: download single file (idempotent)
|
||||
win_get_url:
|
||||
url: https://{{test_win_get_url_host}}
|
||||
dest: '{{test_win_get_url_path}}\web.html'
|
||||
register: http_download_again
|
||||
|
||||
- name: assert download single file (idempotent)
|
||||
assert:
|
||||
that:
|
||||
- not http_download_again is changed
|
||||
|
||||
- name: download single file with force no
|
||||
win_get_url:
|
||||
|
|
@ -53,8 +62,10 @@
|
|||
that:
|
||||
- http_download_no_force is not changed
|
||||
|
||||
- name: manually change last modified time on FTP source to older datetime
|
||||
win_shell: (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
|
||||
- name: manually change the content and last modified time on FTP source to older datetime
|
||||
win_shell: |
|
||||
Set-Content -Path '{{test_win_get_url_path}}\web.html' -Value 'abc'
|
||||
(Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
|
||||
|
||||
- name: download newer file with force no
|
||||
win_get_url:
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingEmptyCatchBlock
|
|||
lib/ansible/modules/windows/win_find.ps1 PSAvoidUsingWMICmdlet
|
||||
lib/ansible/modules/windows/win_firewall_rule.ps1 PSAvoidUsingCmdletAliases
|
||||
lib/ansible/modules/windows/win_firewall_rule.ps1 PSUseApprovedVerbs
|
||||
lib/ansible/modules/windows/win_get_url.ps1 PSUseApprovedVerbs
|
||||
lib/ansible/modules/windows/win_get_url.ps1 PSUsePSCredentialType # Credential param can take a base64 encoded string as well as a PSCredential
|
||||
lib/ansible/modules/windows/win_hotfix.ps1 PSUseApprovedVerbs
|
||||
lib/ansible/modules/windows/win_iis_webbinding.ps1 PSUseApprovedVerbs
|
||||
lib/ansible/modules/windows/win_iis_website.ps1 PSAvoidUsingCmdletAliases
|
||||
|
|
|
|||
Loading…
Reference in a new issue