Windows - Add common util for web requests (#54759)

* Windows - Add common util for web requests

* Use different method of retrieving options from module arg spec

* Added proper version_added for module options

* Fix linting errors

* Fix proxy issues and updated cred docs

* Fix FTP usage with proxy settings

* Removed uneeded function added in bad rebase

* Fix up client certificate auth

* fix new sanity checks

* Edit http agent code and update porting guide

docs/docsite/rst/porting_guides/porting_guide_2.9.rst

@@ -37,7 +37,7 @@ No notable changes
 Modules
 =======
 
-No notable changes
+* The ``win_get_url`` and ``win_uri`` module now sends requests with a default ``User-Agent`` of ``ansible-httpget``. This can be changed by using the ``http_agent`` key.
 
 
 Modules removed

lib/ansible/module_utils/powershell/Ansible.ModuleUtils.WebRequest.psm1

@@ -0,0 +1,500 @@
+# Copyright (c) 2019 Ansible Project
+# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+Function Get-AnsibleWebRequest {
+    <#
+    .SYNOPSIS
+    Creates a System.Net.WebRequest object based on common URL module options in Ansible.
+
+    .DESCRIPTION
+    Will create a WebRequest based on common input options within Ansible. This can be used manually or with
+    Invoke-WithWebRequest.
+
+    .PARAMETER Uri
+    The URI to create the web request for.
+
+    .PARAMETER Method
+    The protocol method to use, if omitted, will use the default value for the URI protocol specified.
+
+    .PARAMETER FollowRedirects
+    Whether to follow redirect reponses. This is only valid when using a HTTP URI.
+        all - Will follow all redirects
+        none - Will follow no redirects
+        safe - Will only follow redirects when GET or HEAD is used as the Method
+
+    .PARAMETER Headers
+    A hashtable or dictionary of header values to set on the request. This is only valid for a HTTP URI.
+
+    .PARAMETER HttpAgent
+    A string to set for the 'User-Agent' header. This is only valid for a HTTP URI.
+
+    .PARAMETER MaximumRedirection
+    The maximum number of redirections that will be followed. This is only valid for a HTTP URI.
+
+    .PARAMETER Timeout
+    The timeout in seconds that defines how long to wait until the request times out.
+
+    .PARAMETER ValidateCerts
+    Whether to validate SSL certificates, default to True.
+
+    .PARAMETER ClientCert
+    The path to PFX file to use for X509 authentication. This is only valid for a HTTP URI. This path can either
+    be a filesystem path (C:\folder\cert.pfx) or a PSPath to a credential (Cert:\CurrentUser\My\<thumbprint>).
+
+    .PARAMETER ClientCertPassword
+    The password for the PFX certificate if required. This is only valid for a HTTP URI.
+
+    .PARAMETER ForceBasicAuth
+    Whether to set the Basic auth header on the first request instead of when required. This is only valid for a
+    HTTP URI.
+
+    .PARAMETER UrlUsername
+    The username to use for authenticating with the target.
+
+    .PARAMETER UrlPassword
+    The password to use for authenticating with the target.
+
+    .PARAMETER UseDefaultCredential
+    Whether to use the current user's credentials if available. This will only work when using Become, using SSH with
+    password auth, or WinRM with CredSSP or Kerberos with credential delegation.
+
+    .PARAMETER UseProxy
+    Whether to use the default proxy defined in IE (WinINet) for the user or set no proxy at all. This should not
+    be set to True when ProxyUrl is also defined.
+
+    .PARAMETER ProxyUrl
+    An explicit proxy server to use for the request instead of relying on the default proxy in IE. This is only
+    valid for a HTTP URI.
+
+    .PARAMETER ProxyUsername
+    An optional username to use for proxy authentication.
+
+    .PARAMETER ProxyPassword
+    The password for ProxyUsername.
+
+    .PARAMETER ProxyUseDefaultCredential
+    Whether to use the current user's credentials for proxy authentication if available. This will only work when
+    using Become, using SSH with password auth, or WinRM with CredSSP or Kerberos with credential delegation.
+
+    .PARAMETER Module
+    The AnsibleBasic module that can be used as a backup parameter source or a way to return warnings back to the
+    Ansible controller.
+
+    .EXAMPLE
+    $spec = @{
+        options = @{}
+    }
+    $spec.options += $ansible_web_request_options
+    $module = Ansible.Basic.AnsibleModule]::Create($args, $spec)
+
+    $web_request = Get-AnsibleWebRequest -Module $module
+    #>
+    [CmdletBinding()]
+    [OutputType([System.Net.WebRequest])]
+    Param (
+        [Alias("url")]
+        [System.Uri]
+        $Uri,
+
+        [System.String]
+        $Method,
+
+        [Alias("follow_redirects")]
+        [ValidateSet("all", "none", "safe")]
+        [System.String]
+        $FollowRedirects = "safe",
+
+        [System.Collections.IDictionary]
+        $Headers,
+
+        [Alias("http_agent")]
+        [System.String]
+        $HttpAgent = "ansible-httpget",
+
+        [Alias("maximum_redirection")]
+        [System.Int32]
+        $MaximumRedirection = 50,
+
+        [System.Int32]
+        $Timeout = 30,
+
+        [Alias("validate_certs")]
+        [System.Boolean]
+        $ValidateCerts = $true,
+
+        # Credential params
+        [Alias("client_cert")]
+        [System.String]
+        $ClientCert,
+
+        [Alias("client_cert_password")]
+        [System.String]
+        $ClientCertPassword,
+
+        [Alias("force_basic_auth")]
+        [Switch]
+        $ForceBasicAuth,
+
+        [Alias("url_username")]
+        [System.String]
+        $UrlUsername,
+
+        [Alias("url_password")]
+        [System.String]
+        $UrlPassword,
+
+        [Alias("use_default_credential")]
+        [Switch]
+        $UseDefaultCredential,
+
+        # Proxy params
+        [Alias("use_proxy")]
+        [System.Boolean]
+        $UseProxy = $true,
+
+        [Alias("proxy_url")]
+        [System.String]
+        $ProxyUrl,
+
+        [Alias("proxy_username")]
+        [System.String]
+        $ProxyUsername,
+
+        [Alias("proxy_password")]
+        [System.String]
+        $ProxyPassword,
+
+        [Alias("proxy_use_default_credential")]
+        [Switch]
+        $ProxyUseDefaultCredential,
+
+        [ValidateScript({ $_.GetType().FullName -eq 'Ansible.Basic.AnsibleModule' })]
+        [System.Object]
+        $Module
+    )
+
+    # Set module options for parameters unless they were explicitly passed in.
+    if ($Module) {
+        foreach ($param in $PSCmdlet.MyInvocation.MyCommand.Parameters.GetEnumerator()) {
+            if ($PSBoundParameters.ContainsKey($param.Key)) {
+                # Was set explicitly we want to use that value
+                continue
+            }
+
+            foreach ($alias in @($Param.Key) + $param.Value.Aliases) {
+                if ($Module.Params.ContainsKey($alias)) {
+                    $var_value = $Module.Params.$alias -as $param.Value.ParameterType
+                    Set-Variable -Name $param.Key -Value $var_value
+                    break
+                }
+            }
+        }
+    }
+
+    # Disable certificate validation if requested
+    # FUTURE: set this on ServerCertificateValidationCallback of the HttpWebRequest once .NET 4.5 is the minimum
+    if (-not $ValidateCerts) {
+        [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
+    }
+
+    # Enable TLS1.1/TLS1.2 if they're available but disabled (eg. .NET 4.5)
+    $security_protocols = [System.Net.ServicePointManager]::SecurityProtocol -bor [System.Net.SecurityProtocolType]::SystemDefault
+    if ([System.Net.SecurityProtocolType].GetMember("Tls11").Count -gt 0) {
+        $security_protocols = $security_protocols -bor [System.Net.SecurityProtocolType]::Tls11
+    }
+    if ([System.Net.SecurityProtocolType].GetMember("Tls12").Count -gt 0) {
+        $security_protocols = $security_protocols -bor [System.Net.SecurityProtocolType]::Tls12
+    }
+    [System.Net.ServicePointManager]::SecurityProtocol = $security_protocols
+
+    $web_request = [System.Net.WebRequest]::Create($Uri)
+    if ($Method) {
+        $web_request.Method = $Method
+    }
+    $web_request.Timeout = $Timeout * 1000
+
+    if ($UseDefaultCredential -and $web_request -is [System.Net.HttpWebRequest]) {
+        $web_request.UseDefaultCredentials = $true
+    } elseif ($UrlUsername) {
+        if ($ForceBasicAuth) {
+            $auth_value = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f $UrlUsername, $UrlPassword)))
+            $web_request.Headers.Add("Authorization", "Basic $auth_value")
+        } else {
+            $credential = New-Object -TypeName System.Net.NetworkCredential -ArgumentList $UrlUsername, $UrlPassword
+            $web_request.Credentials = $credential
+        }
+    }
+
+    if ($ClientCert) {
+        # Expecting either a filepath or PSPath (Cert:\CurrentUser\My\<thumbprint>)
+        $cert = Get-Item -LiteralPath $ClientCert -ErrorAction SilentlyContinue
+        if ($null -eq $cert) {
+            Write-Error -Message "Client certificate '$ClientCert' does not exist" -Category ObjectNotFound
+            return
+        }
+
+        $crypto_ns = 'System.Security.Cryptography.X509Certificates'
+        if ($cert.PSProvider.Name -ne 'Certificate') {
+            try {
+                $cert = New-Object -TypeName "$crypto_ns.X509Certificate2" -ArgumentList @(
+                    $ClientCert, $ClientCertPassword
+                )
+            } catch [System.Security.Cryptography.CryptographicException] {
+                Write-Error -Message "Failed to read client certificate at '$ClientCert'" -Exception $_.Exception -Category SecurityError
+                return
+            }
+        }
+        $web_request.ClientCertificates = New-Object -TypeName "$crypto_ns.X509Certificate2Collection" -ArgumentList @(
+            $cert
+        )
+    }
+
+    if (-not $UseProxy) {
+        $proxy = $null
+    } elseif ($ProxyUrl) {
+        $proxy = New-Object -TypeName System.Net.WebProxy -ArgumentList $ProxyUrl, $true
+    } else  {
+        $proxy = $web_request.Proxy
+    }
+
+    # $web_request.Proxy may return $null for a FTP web request. We only set the credentials if we have an actual
+    # proxy to work with, otherwise just ignore the credentials property.
+    if ($null -ne $proxy) {
+        if ($ProxyUseDefaultCredential) {
+            # Weird hack, $web_request.Proxy returns an IWebProxy object which only gurantees the Credentials
+            # property. We cannot set UseDefaultCredentials so we just set the Credentials to the
+            # DefaultCredentials in the CredentialCache which does the same thing.
+            $proxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials
+        } elseif ($ProxyUsername) {
+            $proxy.Credentials = New-Object -TypeName System.Net.NetworkCredential -ArgumentList @(
+                $ProxyUsername, $ProxyPassword
+            )
+        } else {
+            $proxy.Credentials = $null
+        }
+
+        $web_request.Proxy = $proxy
+    }
+
+    # Some parameters only apply when dealing with a HttpWebRequest
+    if ($web_request -is [System.Net.HttpWebRequest]) {
+        if ($Headers) {
+            foreach ($header in $Headers.GetEnumerator()) {
+                switch ($header.Key) {
+                    Accept { $web_request.Accept = $header.Value }
+                    Connection { $web_request.Connection = $header.Value }
+                    Content-Length { $web_request.ContentLength = $header.Value }
+                    Content-Type { $web_request.ContentType = $header.Value }
+                    Expect { $web_request.Expect = $header.Value }
+                    Date { $web_request.Date = $header.Value }
+                    Host { $web_request.Host = $header.Value }
+                    If-Modified-Since { $web_request.IfModifiedSince = $header.Value }
+                    Range { $web_request.AddRange($header.Value) }
+                    Referer { $web_request.Referer = $header.Value }
+                    Transfer-Encoding {
+                        $web_request.SendChunked = $true
+                        $web_request.TransferEncoding = $header.Value
+                    }
+                    User-Agent { continue }
+                    default { $web_request.Headers.Add($header.Key, $header.Value) }
+                }
+            }
+        }
+
+        # For backwards compatibility we need to support setting the User-Agent if the header was set in the task.
+        # We just need to make sure that if an explicit http_agent module was set then that takes priority.
+        if ($Headers -and $Headers.ContainsKey("User-Agent")) {
+            if ($HttpAgent -eq $ansible_web_request_options.http_agent.default) {
+                $HttpAgent = $Headers['User-Agent']
+            } elseif ($null -ne $Module) {
+                $Module.Warn("The 'User-Agent' header and the 'http_agent' was set, using the 'http_agent' for web request")
+            }
+        }
+        $web_request.UserAgent = $HttpAgent
+
+        switch ($FollowRedirects) {
+            none { $web_request.AllowAutoRedirect = $false }
+            safe {
+                if ($web_request.Method -in @("GET", "HEAD")) {
+                    $web_request.AllowAutoRedirect = $false
+                } else {
+                    $web_request.AllowAutoRedirect = $true
+                }
+            }
+            all { $web_request.AllowAutoRedirect = $true }
+        }
+
+        if ($MaximumRedirection -eq 0) {
+            $web_request.AllowAutoRedirect = $false
+        } else {
+            $web_request.MaximumAutomaticRedirections = $MaximumRedirection
+        }
+    }
+
+    return $web_request
+}
+
+Function Invoke-WithWebRequest {
+    <#
+    .SYNOPSIS
+    Invokes a ScriptBlock with the WebRequest.
+
+    .DESCRIPTION
+    Invokes the ScriptBlock and handle extra information like accessing the response stream, closing those streams
+    safely as well as setting common module return values.
+
+    .PARAMETER Module
+    The Ansible.Basic module to set the return values for. This will set the following return values;
+        elapsed - The total time, in seconds, that it took to send the web request and process the response
+        msg - The human readable description of the response status code
+        status_code - An int that is the response status code
+
+    .PARAMETER Request
+    The System.Net.WebRequest to call. This can either be manually crafted or created with Get-AnsibleWebRequest.
+
+    .PARAMETER Script
+    The ScriptBlock to invoke during the web request. This ScriptBlock should take in the params
+        Param ([System.Net.WebResponse]$Response, [System.IO.Stream]$Stream)
+
+    This scriptblock should manage the response based on what it need to do.
+
+    .PARAMETER Body
+    An optional Stream to send to the target during the request.
+
+    .PARAMETER IgnoreBadResponse
+    By default a WebException will be raised for a non 2xx status code and the Script will not be invoked. This
+    parameter can be set to process all responses regardless of the status code.
+
+    .EXAMPLE Basic module that downloads a file
+    $spec = @{
+        options = @{
+            path = @{ type = "path"; required = $true }
+        }
+    }
+    $spec.options += $ansible_web_request_options
+    $module = Ansible.Basic.AnsibleModule]::Create($args, $spec)
+
+    $web_request = Get-AnsibleWebRequest -Module $module
+
+    Invoke-WithWebRequest -Module $module -Request $web_request -Script {
+        Param ([System.Net.WebResponse]$Response, [System.IO.Stream]$Stream)
+
+        $fs = [System.IO.File]::Create($module.Params.path)
+        try {
+            $Stream.CopyTo($fs)
+            $fs.Flush()
+        } finally {
+            $fs.Dispose()
+        }
+    }
+    #>
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory=$true)]
+        [System.Object]
+        [ValidateScript({ $_.GetType().FullName -eq 'Ansible.Basic.AnsibleModule' })]
+        $Module,
+
+        [Parameter(Mandatory=$true)]
+        [System.Net.WebRequest]
+        $Request,
+
+        [Parameter(Mandatory=$true)]
+        [ScriptBlock]
+        $Script,
+
+        [AllowNull()]
+        [System.IO.Stream]
+        $Body,
+
+        [Switch]
+        $IgnoreBadResponse
+    )
+
+    $start = Get-Date
+    if ($null -ne $Body) {
+        $request_st = $Request.GetRequestStream()
+        try {
+            $Body.CopyTo($request_st)
+            $request_st.Flush()
+        } finally {
+            $request_st.Close()
+        }
+    }
+
+    try {
+        try {
+            $web_response = $Request.GetResponse()
+        } catch [System.Net.WebException] {
+            # A WebResponse with a status code not in the 200 range will raise a WebException. We check if the
+            # exception raised contains the actual response and continue on if IgnoreBadResponse is set. We also
+            # make sure we set the status_code return value on the Module object if possible
+
+            if ($_.Exception.PSObject.Properties.Name -match "Response") {
+                $web_response = $_.Exception.Response
+
+                if (-not $IgnoreBadResponse -or $null -eq $web_response) {
+                    $Module.Result.msg = $_.Exception.StatusDescription
+                    $Module.Result.status_code = $_.Exception.Response.StatusCode
+                    throw $_
+                }
+            } else {
+                throw $_
+            }
+        }
+
+        if ($Request.RequestUri.IsFile) {
+            # A FileWebResponse won't have these properties set
+            $Module.Result.msg = "OK"
+            $Module.Result.status_code = 200
+        } else {
+            $Module.Result.msg = $web_response.StatusDescription
+            $Module.Result.status_code = $web_response.StatusCode
+        }
+
+        $response_stream = $web_response.GetResponseStream()
+        try {
+            # Invoke the ScriptBlock and pass in WebResponse and ResponseStream
+            &$Script -Response $web_response -Stream $response_stream
+        } finally {
+            $response_stream.Dispose()
+        }
+    } finally {
+        if ($web_response) {
+            $web_response.Close()
+        }
+        $Module.Result.elapsed = ((Get-date) - $start).TotalSeconds
+    }
+}
+
+$ansible_web_request_options = @{
+    url = @{ type="str"; required=$true }
+    method = @{ type="str" }
+    follow_redirects = @{ type="str"; choices=@("all","none","safe"); default="safe" }
+    headers = @{ type="dict" }
+    http_agent = @{ type="str"; default="ansible-httpget" }
+    maximum_redirection = @{ type="int"; default=50 }
+    timeout = @{ type="int"; default=30 }  # Was defaulted to 10 in win_get_url but 30 in win_uri so we use 30
+    validate_certs = @{ type="bool"; default=$true }
+
+    # Credential options
+    client_cert = @{ type="str" }
+    client_cert_password = @{ type="str"; no_log=$true }
+    force_basic_auth = @{ type="bool"; default=$false }
+    url_username = @{ type="str"; aliases=@("user", "username") }  # user was used in win_uri
+    url_password = @{ type="str"; aliases=@("password"); no_log=$true }
+    use_default_credential = @{ type="bool"; default=$false }
+
+    # Proxy options
+    use_proxy = @{ type="bool"; default=$true }
+    proxy_url = @{ type="str" }
+    proxy_username = @{ type="str" }
+    proxy_password = @{ type="str"; no_log=$true }
+    proxy_use_default_credential = @{ type="bool"; default=$false }
+}
+
+$export_members = @{
+    Function = "Get-AnsibleWebRequest", "Invoke-WithWebRequest"
+    Variable = "ansible_web_request_options"
+}
+Export-ModuleMember @export_members

lib/ansible/modules/windows/win_get_url.ps1

@@ -8,23 +8,12 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 #AnsibleRequires -CSharpUtil Ansible.Basic
-#Requires -Module Ansible.ModuleUtils.AddType
 #Requires -Module Ansible.ModuleUtils.FileUtil
+#Requires -Module Ansible.ModuleUtils.WebRequest
 
 $spec = @{
     options = @{
-        url = @{ type='str'; required=$true }
         dest = @{ type='path'; required=$true }
-        timeout = @{ type='int'; default=10 }
-        headers = @{ type='dict'; default=@{} }
-        validate_certs = @{ type='bool'; default=$true }
-        url_username = @{ type='str'; aliases=@( 'username' ) }
-        url_password = @{ type='str'; aliases=@( 'password' ); no_log=$true }
-        force_basic_auth = @{ type='bool'; default=$false }
-        use_proxy = @{ type='bool'; default=$true }
-        proxy_url = @{ type='str' }
-        proxy_username = @{ type='str' }
-        proxy_password = @{ type='str'; no_log=$true }
         force = @{ type='bool'; default=$true }
         checksum = @{ type='str' }
         checksum_algorithm = @{ type='str'; default='sha1'; choices = @("md5", "sha1", "sha256", "sha384", "sha512") }
@@ -35,21 +24,12 @@ $spec = @{
     )
     supports_check_mode = $true
 }
+$spec.options += $ansible_web_request_options
 
 $module = [Ansible.Basic.AnsibleModule]::Create($args, $spec)
 
 $url = $module.Params.url
 $dest = $module.Params.dest
-$timeout = $module.Params.timeout
-$headers = $module.Params.headers
-$validate_certs = $module.Params.validate_certs
-$url_username = $module.Params.url_username
-$url_password = $module.Params.url_password
-$force_basic_auth = $module.Params.force_basic_auth
-$use_proxy = $module.Params.use_proxy
-$proxy_url = $module.Params.proxy_url
-$proxy_username = $module.Params.proxy_username
-$proxy_password = $module.Params.proxy_password
 $force = $module.Params.force
 $checksum = $module.Params.checksum
 $checksum_algorithm = $module.Params.checksum_algorithm
@@ -58,103 +38,11 @@ $checksum_url = $module.Params.checksum_url
 $module.Result.elapsed = 0
 $module.Result.url = $url
 
-Function Invoke-AnsibleWebRequest {
-    <#
-    .SYNOPSIS
-    Creates a WebRequest and invokes a ScriptBlock with the response passed in.
-    It handles the common module options like credential, timeout, proxy options
-    in a single location to reduce code duplication.
-    #>
-    param(
-        [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
-        [Parameter(Mandatory=$true)][Uri]$Uri,
-        [Parameter(Mandatory=$true)][Hashtable]$Method,
-        [Parameter(Mandatory=$true)][ScriptBlock]$Script, # Invoked in this cmdlet
-        [System.Collections.IDictionary]$Headers,
-        [Int32]$Timeout,
-        [Switch]$UseProxy,
-        [System.Net.WebProxy]$Proxy,
-        $Credential  # Either a String (force_basic_auth) or NetCredentials
-    )
-
-    $web_request = [System.Net.WebRequest]::Create($Uri)
-    $web_request.Method = $Method.($web_request.GetType().Name)
-
-    foreach ($header in $headers.GetEnumerator()) {
-        # some headers need to be set on the property itself
-        switch ($header.Key) {
-            Accept { $web_request.Accept = $header.Value }
-            Connection { $web_request.Connection = $header.Value }
-            Content-Length { $web_request.ContentLength = $header.Value }
-            Content-Type { $web_request.ContentType = $header.Value }
-            Expect { $web_request.Expect = $header.Value }
-            Date { $web_request.Date = $header.Value }
-            Host { $web_request.Host = $header.Value }
-            If-Modified-Since { $web_request.IfModifiedSince = $header.Value }
-            Range { $web_request.AddRange($header.Value) }
-            Referer { $web_request.Referer = $header.Value }
-            Transfer-Encoding {
-                $web_request.SendChunked = $true
-                $web_request.TransferEncoding = $header.Value
-            }
-            User-Agent { $web_request.UserAgent = $header.Value }
-            default { $web_request.Headers.Add($header.Key, $header.Value) }
-        }
-    }
-
-    if ($timeout) {
-        $web_request.Timeout = $timeout * 1000
-    }
-
-    if (-not $UseProxy) {
-        $web_request.Proxy = $null
-    } elseif ($Proxy) {
-        $web_request.Proxy = $Proxy
-    }
-
-    if ($Credential) {
-        if ($Credential -is [String]) {
-            # force_basic_auth=yes is set
-            $web_request.Headers.Add("Authorization", "Basic $Credential")
-        } else {
-            $web_request.Credentials = $Credential
-        }
-    }
-
-    try {
-        $web_response = $web_request.GetResponse()
-        $response_stream = $web_response.GetResponseStream()
-        try {
-            # Invoke the ScriptBlock and pass in the WebResponse and ResponseStream
-            &$Script -Response $web_response -Stream $response_stream
-        } finally {
-            $response_stream.Dispose()
-        }
-
-        if ($Uri.IsFile) {
-            # A FileWebResponse won't have these properties set
-            $module.Result.msg = "OK"
-            $module.Result.status_code = 200
-        } else {
-            $module.Result.msg = [string]$web_response.StatusDescription
-            $module.Result.status_code = [int]$web_response.StatusCode
-        }
-    } catch [System.Net.WebException] {
-        $module.Result.status_code = [int]$_.Exception.Response.StatusCode
-        $module.FailJson("Error requesting '$Uri'. $($_.Exception.Message)", $_)
-    } finally {
-        if ($web_response) {
-            $web_response.Close()
-        }
-    }
-}
-
 Function Get-ChecksumFromUri {
     param(
         [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
         [Parameter(Mandatory=$true)][Uri]$Uri,
-        [Uri]$SourceUri,
+        [Uri]$SourceUri
-        [Hashtable]$RequestParams
     )
 
     $script = {
@@ -176,18 +64,10 @@ Function Get-ChecksumFromUri {
 
         Write-Output -InputObject $hash_from_file
     }
-    $invoke_args = @{
+    $web_request = Get-AnsibleWebRequest -Uri $Uri -Module $Module
-        Module = $Module
+
-        Uri = $Uri
-        Method = @{
-            FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile
-            FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile
-            HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get
-        }
-        Script = $script
-    }
     try {
-        Invoke-AnsibleWebRequest @invoke_args @RequestParams
+        Invoke-WithWebRequest -Module $Module -Request $web_request -Script $script
     } catch {
         $Module.FailJson("Error when getting the remote checksum from '$Uri'. $($_.Exception.Message)", $_)
     }
@@ -203,8 +83,7 @@ Function Compare-ModifiedFile {
     param(
         [Parameter(Mandatory=$true)][Ansible.Basic.AnsibleModule]$Module,
         [Parameter(Mandatory=$true)][Uri]$Uri,
-        [Parameter(Mandatory=$true)][String]$Dest,
+        [Parameter(Mandatory=$true)][String]$Dest
-        [Hashtable]$RequestParams
     )
 
     $dest_last_mod = (Get-AnsibleItem -Path $Dest).LastWriteTimeUtc
@@ -213,17 +92,15 @@ Function Compare-ModifiedFile {
     if ($Uri.IsFile) {
         $src_last_mod = (Get-AnsibleItem -Path $Uri.AbsolutePath).LastWriteTimeUtc
     } else {
-        $invoke_args = @{
+        $web_request = Get-AnsibleWebRequest -Uri $Uri -Module $Module
-            Module = $Module
+        $web_request.Method = switch ($web_request.GetType().Name) {
-            Uri = $Uri
+            FtpWebRequest { [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp }
-            Method = @{
+            HttpWebRequest { [System.Net.WebRequestMethods+Http]::Head }
-                FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::GetDateTimestamp
-                HttpWebRequest = [System.Net.WebRequestMethods+Http]::Head
-            }
-            Script = { param($Response, $Stream); $Response.LastModified }
         }
+        $script = { param($Response, $Stream); $Response.LastModified }
+
         try {
-            $src_last_mod = Invoke-AnsibleWebRequest @invoke_args @RequestParams
+            $src_last_mod = Invoke-WithWebRequest -Module $Module -Request $web_request -Script $script
         } catch {
             $Module.FailJson("Error when requesting 'Last-Modified' date from '$Uri'. $($_.Exception.Message)", $_)
         }
@@ -263,8 +140,7 @@ Function Invoke-DownloadFile {
         [Parameter(Mandatory=$true)][Uri]$Uri,
         [Parameter(Mandatory=$true)][String]$Dest,
         [String]$Checksum,
-        [String]$ChecksumAlgorithm,
+        [String]$ChecksumAlgorithm
-        [Hashtable]$RequestParams
     )
 
     # Check $dest parent folder exists before attempting download, which avoids unhelpful generic error message.
@@ -312,54 +188,15 @@ Function Invoke-DownloadFile {
             $Module.Result.changed = $true
         }
     }
+    $web_request = Get-AnsibleWebRequest -Uri $Uri -Module $Module
 
-    $invoke_args = @{
-        Module = $Module
-        Uri = $Uri
-        Method = @{
-            FileWebRequest = [System.Net.WebRequestMethods+File]::DownloadFile
-            FtpWebRequest = [System.Net.WebRequestMethods+Ftp]::DownloadFile
-            HttpWebRequest = [System.Net.WebRequestMethods+Http]::Get
-        }
-        Script = $download_script
-    }
-
-    $module_start = Get-Date
     try {
-        Invoke-AnsibleWebRequest @invoke_args @RequestParams
+        Invoke-WithWebRequest -Module $Module -Request $web_request -Script $download_script
     } catch {
-        $Module.FailJson("Unknown error downloading '$Uri' to '$Dest': $($_.Exception.Message)", $_)
+        $Module.FailJson("Error downloading '$Uri' to '$Dest': $($_.Exception.Message)", $_)
-    } finally {
-        $Module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
     }
 }
 
-if (-not $use_proxy -and ($proxy_url -or $proxy_username -or $proxy_password)) {
-    $module.Warn("Not using a proxy on request, however a 'proxy_url', 'proxy_username' or 'proxy_password' was defined.")
-}
-
-$proxy = $null
-if ($proxy_url) {
-    $proxy = New-Object System.Net.WebProxy($proxy_url, $true)
-    if ($proxy_username -and $proxy_password) {
-        $proxy_credential = New-Object System.Net.NetworkCredential($proxy_username, $proxy_password)
-        $proxy.Credentials = $proxy_credential
-    }
-}
-
-$credentials = $null
-if ($url_username) {
-    if ($force_basic_auth) {
-        $credentials = [convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes($url_username+":"+$url_password))
-    } else {
-        $credentials = New-Object System.Net.NetworkCredential($url_username, $url_password)
-    }
-}
-
-if (-not $validate_certs) {
-    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
-}
-
 # Use last part of url for dest file name if a directory is supplied for $dest
 if (Test-Path -LiteralPath $dest -PathType Container) {
     $uri = [System.Uri]$url
@@ -380,24 +217,6 @@ if (Test-Path -LiteralPath $dest -PathType Container) {
 
 $module.Result.dest = $dest
 
-# Enable TLS1.1/TLS1.2 if they're available but disabled (eg. .NET 4.5)
-$security_protocols = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::SystemDefault
-if ([Net.SecurityProtocolType].GetMember("Tls11").Count -gt 0) {
-    $security_protocols = $security_protocols -bor [Net.SecurityProtocolType]::Tls11
-}
-if ([Net.SecurityProtocolType].GetMember("Tls12").Count -gt 0) {
-    $security_protocols = $security_protocols -bor [Net.SecurityProtocolType]::Tls12
-}
-[Net.ServicePointManager]::SecurityProtocol = $security_protocols
-
-$request_params = @{
-    Credential = $credentials
-    Headers = $headers
-    Timeout = $timeout
-    UseProxy = $use_proxy
-    Proxy = $proxy
-}
-
 if ($checksum) {
     $checksum = $checksum.Trim().ToLower()
 }
@@ -415,20 +234,20 @@ if ($checksum_url) {
         $module.FailJson("Unsupported 'checksum_url' value for '$dest': '$checksum_url'")
     }
 
-    $checksum = Get-ChecksumFromUri -Module $Module -Uri $checksum_uri -SourceUri $url -RequestParams $request_params
+    $checksum = Get-ChecksumFromUri -Module $Module -Uri $checksum_uri -SourceUri $url
 }
 
 if ($force -or -not (Test-Path -LiteralPath $dest)) {
     # force=yes or dest does not exist, download the file
     # Note: Invoke-DownloadFile will compare the checksums internally if dest exists
     Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum `
-        -ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params
+        -ChecksumAlgorithm $checksum_algorithm
 } else {
     # force=no, we want to check the last modified dates and only download if they don't match
-    $is_modified = Compare-ModifiedFile -Module $module -Uri $url -Dest $dest -RequestParams $request_params
+    $is_modified = Compare-ModifiedFile -Module $module -Uri $url -Dest $dest
     if ($is_modified) {
         Invoke-DownloadFile -Module $module -Uri $url -Dest $dest -Checksum $checksum `
-            -ChecksumAlgorithm $checksum_algorithm -RequestParams $request_params
+            -ChecksumAlgorithm $checksum_algorithm
    }
 }
 

lib/ansible/modules/windows/win_get_url.py

@@ -43,36 +43,6 @@ options:
     type: bool
     default: yes
     version_added: "2.0"
-  headers:
-    description:
-    - Add custom HTTP headers to a request (as a dictionary).
-    type: dict
-    version_added: '2.4'
-  url_username:
-    description:
-    - Basic authentication username.
-    type: str
-    aliases: [ username ]
-  url_password:
-    description:
-    - Basic authentication password.
-    type: str
-    aliases: [ password ]
-  force_basic_auth:
-    description:
-    - If C(yes), will add a Basic authentication header on the initial request.
-    - If C(no), will use Microsoft's WebClient to handle authentication.
-    type: bool
-    default: no
-    version_added: "2.5"
-  validate_certs:
-    description:
-    - If C(no), SSL certificates will not be validated. This should only be used
-      on personally controlled sites using self-signed certificates.
-    - If C(skip_certificate_validation) was set, it overrides this option.
-    type: bool
-    default: yes
-    version_added: '2.4'
   checksum:
     description:
       - If a I(checksum) is passed to this parameter, the digest of the
@@ -104,33 +74,27 @@ options:
     type: str
     version_added: "2.8"
   proxy_url:
-    description:
-    - The full URL of the proxy server to download through.
-    type: str
     version_added: "2.0"
   proxy_username:
-    description:
-    - Proxy authentication username.
-    type: str
     version_added: "2.0"
   proxy_password:
-    description:
-    - Proxy authentication password.
-    type: str
     version_added: "2.0"
+  headers:
+    version_added: "2.4"
   use_proxy:
+    version_added: "2.4"
+  follow_redirects:
+    version_added: "2.9"
+  maximum_redirection:
+    version_added: "2.9"
+  client_cert:
+    version_added: "2.9"
+  client_cert_password:
+    version_added: "2.9"
+  method:
     description:
-    - If C(no), it will not use a proxy, even if one is defined in an environment
+    - This option is not for use with C(win_get_url) and should be ignored.
-      variable on the target hosts.
+    version_added: "2.9"
-    type: bool
-    default: yes
-    version_added: '2.4'
-  timeout:
-    description:
-    - Timeout in seconds for URL request.
-    type: int
-    default: 10
-    version_added : '2.4'
 notes:
 - If your URL includes an escaped slash character (%2F) this module will convert it to a real slash.
   This is a result of the behaviour of the System.Uri class as described in
@@ -138,6 +102,8 @@ notes:
 - Since Ansible 2.8, the module will skip reporting a change if the remote
   checksum is the same as the local local even when C(force=yes). This is to
   better align with M(get_url).
+extends_documentation_fragment:
+- url_windows
 seealso:
 - module: get_url
 - module: uri

lib/ansible/modules/windows/win_uri.ps1

@@ -8,59 +8,34 @@
 #Requires -Module Ansible.ModuleUtils.CamelConversion
 #Requires -Module Ansible.ModuleUtils.FileUtil
 #Requires -Module Ansible.ModuleUtils.Legacy
+#Requires -Module Ansible.ModuleUtils.WebRequest
 
 $spec = @{
     options = @{
-        url = @{ type = "str"; required = $true }
-        method = @{
-            type = "str"
-            default = "GET"
-       }
        content_type = @{ type = "str" }
-       headers = @{ type = "dict" }
        body = @{ type = "raw" }
        dest = @{ type = "path" }
-       user = @{ type = "str" }
-       password = @{ type = "str"; no_log = $true }
-       force_basic_auth = @{ type = "bool"; default = $false }
        creates = @{ type = "path" }
        removes = @{ type = "path" }
-       follow_redirects = @{
-           type = "str"
-           default = "safe"
-           choices = "all", "none", "safe"
-       }
-       maximum_redirection = @{ type = "int"; default = 50 }
        return_content = @{ type = "bool"; default = $false }
        status_code = @{ type = "list"; elements = "int"; default = @(200) }
-       timeout = @{ type = "int"; default = 30 }
-       validate_certs = @{ type = "bool"; default = $true }
-       client_cert = @{ type = "path" }
-       client_cert_password = @{ type = "str"; no_log = $true }
     }
     supports_check_mode = $true
 }
+$spec.options += $ansible_web_request_options
+$spec.options.method.default = "GET"
+
 $module = [Ansible.Basic.AnsibleModule]::Create($args, $spec)
 
 $url = $module.Params.url
 $method = $module.Params.method.ToUpper()
 $content_type = $module.Params.content_type
-$headers = $module.Params.headers
 $body = $module.Params.body
 $dest = $module.Params.dest
-$user = $module.Params.user
-$password = $module.Params.password
-$force_basic_auth = $module.Params.force_basic_auth
 $creates = $module.Params.creates
 $removes = $module.Params.removes
-$follow_redirects = $module.Params.follow_redirects
-$maximum_redirection = $module.Params.maximum_redirection
 $return_content = $module.Params.return_content
 $status_code = $module.Params.status_code
-$timeout = $module.Params.timeout
-$validate_certs = $module.Params.validate_certs
-$client_cert = $module.Params.client_cert
-$client_cert_password = $module.Params.client_cert_password
 
 $JSON_CANDIDATES = @('text', 'json', 'javascript')
 
@@ -83,152 +58,16 @@ if ($removes -and -not (Test-AnsiblePath -Path $removes)) {
     $module.ExitJson()
 }
 
-# Enable TLS1.1/TLS1.2 if they're available but disabled (eg. .NET 4.5)
+$client = Get-AnsibleWebRequest -Module $module
-$security_protocols = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::SystemDefault
-if ([Net.SecurityProtocolType].GetMember("Tls11").Count -gt 0) {
-    $security_protocols = $security_protocols -bor [Net.SecurityProtocolType]::Tls11
-}
-if ([Net.SecurityProtocolType].GetMember("Tls12").Count -gt 0) {
-    $security_protocols = $security_protocols -bor [Net.SecurityProtocolType]::Tls12
-}
-[Net.ServicePointManager]::SecurityProtocol = $security_protocols
-
-$client = [System.Net.WebRequest]::Create($url)
-$client.Method = $method
-$client.Timeout = $timeout * 1000
-
-# Disable redirection if requested
-switch($follow_redirects) {
-    "none" {
-        $client.AllowAutoRedirect = $false
-    }
-    "safe" {
-        if (@("GET", "HEAD") -notcontains $method) {
-            $client.AllowAutoRedirect = $false
-        } else {
-            $client.AllowAutoRedirect = $true
-        }
-    }
-    default {
-        $client.AllowAutoRedirect = $true
-    }
-}
-if ($maximum_redirection -eq 0) {
-    # 0 is not a valid option, need to disable redirection through AllowAutoRedirect
-    $client.AllowAutoRedirect = $false
-} else {
-    $client.MaximumAutomaticRedirections = $maximum_redirection
-}
-
-if (-not $validate_certs) {
-    [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true }
-}
 
 if ($null -ne $content_type) {
     $client.ContentType = $content_type
 }
 
-if ($headers) {
+$response_script = {
-    $req_headers = New-Object -TypeName System.Net.WebHeaderCollection
+    param($Response, $Stream)
-    foreach ($header in $headers.GetEnumerator()) {
-        # some headers need to be set on the property itself
-        switch ($header.Key) {
-            Accept { $client.Accept = $header.Value }
-            Connection { $client.Connection = $header.Value }
-            Content-Length { $client.ContentLength = $header.Value }
-            Content-Type { $client.ContentType = $header.Value }
-            Expect { $client.Expect = $header.Value }
-            Date { $client.Date = $header.Value }
-            Host { $client.Host = $header.Value }
-            If-Modified-Since { $client.IfModifiedSince = $header.Value }
-            Range { $client.AddRange($header.Value) }
-            Referer { $client.Referer = $header.Value }
-            Transfer-Encoding {
-                $client.SendChunked = $true
-                $client.TransferEncoding = $header.Value
-            }
-            User-Agent { $client.UserAgent = $header.Value }
-            default { $req_headers.Add($header.Key, $header.Value) }
-        }
-    }
-    $client.Headers.Add($req_headers)
-}
 
-if ($client_cert) {
+    ForEach ($prop in $Response.PSObject.Properties) {
-    if (-not (Test-AnsiblePath -Path $client_cert)) {
-        $module.FailJson("Client certificate '$client_cert' does not exit")
-    }
-    try {
-        $certs = New-Object -TypeName System.Security.Cryptography.X509Certificates.X509Certificate2Collection -ArgumentList $client_cert, $client_cert_password
-        $client.ClientCertificates = $certs
-    } catch [System.Security.Cryptography.CryptographicException] {
-        $module.FailJson("Failed to read client certificate '$client_cert': $($_.Exception.Message)", $_)
-    } catch {
-        $module.FailJson("Unhandled exception when reading client certificate at '$client_cert': $($_.Exception.Message)", $_)
-    }
-}
-
-if ($user -and $password) {
-    if ($force_basic_auth) {
-        $basic_value = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes("$($user):$($password)"))
-        $client.Headers.Add("Authorization", "Basic $basic_value")
-    } else {
-        $sec_password = ConvertTo-SecureString -String $password -AsPlainText -Force
-        $credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $user, $sec_password
-        $client.Credentials = $credential
-    }
-} elseif ($user -or $password) {
-    $module.Warn("Both 'user' and 'password' parameters are required together, skipping authentication")
-}
-
-if ($null -ne $body) {
-    if ($body -is [System.Collections.IDictionary] -or $body -is [System.Collections.IList]) {
-        $body_string = ConvertTo-Json -InputObject $body -Compress
-    } elseif ($body -isnot [String]) {
-        $body_string = $body.ToString()
-    } else {
-        $body_string = $body
-    }
-    $buffer = [System.Text.Encoding]::UTF8.GetBytes($body_string)
-
-    $req_st = $client.GetRequestStream()
-    try {
-        $req_st.Write($buffer, 0, $buffer.Length)
-    } finally {
-        $req_st.Flush()
-        $req_st.Close()
-    }
-}
-
-$module_start = Get-Date
-
-try {
-    $response = $client.GetResponse()
-} catch [System.Net.WebException] {
-    $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-    $response = $null
-    if ($_.Exception.PSObject.Properties.Name -match "Response") {
-        # was a non-successful response but we at least have a response and
-        # should parse it below according to module input
-        $response = $_.Exception.Response
-    }
-
-    # in the case a response (or empty response) was on the exception like in
-    # a timeout scenario, we should still fail
-    if ($null -eq $response) {
-        $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-        $module.FailJson("WebException occurred when sending web request: $($_.Exception.Message)", $_)
-    }
-} catch [System.Net.ProtocolViolationException] {
-    $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-    $module.FailJson("ProtocolViolationException when sending web request: $($_.Exception.Message)", $_)
-} catch {
-    $module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-    $module.FailJson("Unhandled exception occured when sending web request. Exception: $($_.Exception.Message)", $_)
-}
-$module.Result.elapsed = ((Get-Date) - $module_start).TotalSeconds
-
-ForEach ($prop in $response.psobject.properties) {
         $result_key = Convert-StringToSnakeCase -string $prop.Name
         $prop_value = $prop.Value
         # convert and DateTime values to ISO 8601 standard
@@ -239,8 +78,8 @@ ForEach ($prop in $response.psobject.properties) {
     }
 
     # manually get the headers as not all of them are in the response properties
-foreach ($header_key in $response.Headers.GetEnumerator()) {
+    foreach ($header_key in $Response.Headers.GetEnumerator()) {
-    $header_value = $response.Headers[$header_key]
+        $header_value = $Response.Headers[$header_key]
         $header_key = $header_key.Replace("-", "") # replace - with _ for snake conversion
         $header_key = Convert-StringToSnakeCase -string $header_key
         $module.Result.$header_key = $header_value
@@ -248,13 +87,10 @@ foreach ($header_key in $response.Headers.GetEnumerator()) {
 
     # we only care about the return body if we need to return the content or create a file
     if ($return_content -or $dest) {
-    $resp_st = $response.GetResponseStream()
-
         # copy to a MemoryStream so we can read it multiple times
         $memory_st = New-Object -TypeName System.IO.MemoryStream
         try {
-        $resp_st.CopyTo($memory_st)
+            $Stream.CopyTo($memory_st)
-        $resp_st.Close()
 
             if ($return_content) {
                 $memory_st.Seek(0, [System.IO.SeekOrigin]::Begin) > $null
@@ -301,8 +137,33 @@ if ($return_content -or $dest) {
         }
     }
 
-if ($status_code -notcontains $response.StatusCode) {
+    if ($status_code -notcontains $Response.StatusCode) {
-    $module.FailJson("Status code of request '$([int]$response.StatusCode)' is not in list of valid status codes $status_code : $($response.StatusCode)'.")
+        $module.FailJson("Status code of request '$([int]$Response.StatusCode)' is not in list of valid status codes $status_code : $($Response.StatusCode)'.")
+    }
+}
+
+$body_st = $null
+if ($null -ne $body) {
+    if ($body -is [System.Collections.IDictionary] -or $body -is [System.Collections.IList]) {
+        $body_string = ConvertTo-Json -InputObject $body -Compress
+    } elseif ($body -isnot [String]) {
+        $body_string = $body.ToString()
+    } else {
+        $body_string = $body
+    }
+    $buffer = [System.Text.Encoding]::UTF8.GetBytes($body_string)
+
+    $body_st = New-Object -TypeName System.IO.MemoryStream -ArgumentList @(,$buffer)
+}
+
+try {
+    Invoke-WithWebRequest -Module $module -Request $client -Script $response_script -Body $body_st -IgnoreBadResponse
+} catch {
+    $module.FailJson("Unhandled exception occurred when sending web request. Exception: $($_.Exception.Message)", $_)
+} finally {
+    if ($null -ne $body_st) {
+        $body_st.Dispose()
+    }
 }
 
 $module.ExitJson()

lib/ansible/modules/windows/win_uri.py

@@ -37,36 +37,11 @@ options:
     description:
     - The body of the HTTP request/response to the web service.
     type: raw
-  user:
-    description:
-    - Username to use for authentication.
-    type: str
-    version_added: '2.4'
-  password:
-    description:
-    - Password to use for authentication.
-    type: str
-    version_added: '2.4'
-  force_basic_auth:
-    description:
-    - By default the authentication information is only sent when a webservice
-      responds to an initial request with a 401 status. Since some basic auth
-      services do not properly send a 401, logins will fail.
-    - This option forces the sending of the Basic authentication header upon
-      the initial request.
-    type: bool
-    default: no
-    version_added: '2.5'
   dest:
     description:
     - Output the response body to a file.
     type: path
     version_added: '2.3'
-  headers:
-    description:
-    - Extra headers to set on the request, see the examples for more details on
-      how to set this.
-    type: dict
   creates:
     description:
     - A filename, when it already exists, this step will be skipped.
@@ -93,61 +68,36 @@ options:
     type: list
     default: [ 200 ]
     version_added: '2.4'
-  timeout:
+  url_username:
     description:
-    - Specifies how long the request can be pending before it times out (in seconds).
+    - The username to use for authentication.
-    - The value 0 (zero) specifies an indefinite time-out.
+    - Was originally called I(user) but was changed to I(url_username) in
-    - A Domain Name System (DNS) query can take up to 15 seconds to return or time out.
+      Ansible 2.9.
-      If your request contains a host name that requires resolution, and you set
+    version_added: "2.4"
-      C(timeout) to a value greater than zero, but less than 15 seconds, it can
+  url_password:
-      take 15 seconds or more before your request times out.
+    description:
-    type: int
+    - The password for I(url_username).
-    default: 30
+    - Was originally called I(password) but was changed to I(url_password) in
-    version_added: '2.4'
+      Ansible 2.9.
+    version_added: "2.4"
   follow_redirects:
-    description:
+    version_added: "2.4"
-     - Whether or not the C(win_uri) module should follow redirects.
-     - C(all) will follow all redirects.
-     - C(none) will not follow any redirects.
-     - C(safe) will follow only "safe" redirects, where "safe" means that the client is only
-       doing a C(GET) or C(HEAD) on the URI to which it is being redirected.
-    type: str
-    choices: [ all, none, safe ]
-    default: safe
-    version_added: '2.4'
   maximum_redirection:
-    description:
+    version_added: "2.4"
-    - Specifies how many times C(win_uri) redirects a connection to an alternate
-      Uniform Resource Identifier (URI) before the connection fails.
-    - If C(maximum_redirection) is set to 0 (zero)
-      or C(follow_redirects) is set to C(none),
-      or set to C(safe) when not doing C(GET) or C(HEAD) it prevents all redirection.
-    type: int
-    default: 50
-    version_added: '2.4'
-  validate_certs:
-    description:
-    - If C(no), SSL certificates will not be validated.  This should only
-      set to C(no) used on personally controlled sites using self-signed
-      certificates.
-    type: bool
-    default: yes
-    version_added: '2.4'
   client_cert:
-    description:
+    version_added: "2.4"
-    - Specifies the client certificate (.pfx) that is used for a secure web request.
-    - The WinRM connection must be authenticated with C(CredSSP) if the
-      certificate file is not password protected.
-    - Other authentication types can set I(client_cert_password) when the cert
-      is password protected.
-    type: path
-    version_added: '2.4'
   client_cert_password:
-    description:
+    version_added: "2.5"
-    - The password for the client certificate (.pfx) file that is used for a
+  use_proxy:
-      secure web request.
+    version_added: "2.9"
-    type: str
+  proxy_url:
-    version_added: '2.5'
+    version_added: "2.9"
+  proxy_username:
+    version_added: "2.9"
+  proxy_password:
+    version_added: "2.9"
+extends_documentation_fragment:
+- url_windows
 seealso:
 - module: uri
 - module: win_get_url

lib/ansible/plugins/doc_fragments/url_windows.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+
+    # Standard files documentation fragment
+    DOCUMENTATION = r'''
+options:
+  url:
+    description:
+    - The URL to make the request with.
+    required: yes
+    type: str
+  method:
+    description:
+    - The HTTP Method of the request.
+    type: str
+  follow_redirects:
+    description:
+    - Whether or the module should follow redirects.
+    - C(all) will follow all redirect.
+    - C(none) will not follow any redirect.
+    - C(safe) will follow only "safe" redirects, where "safe" means that the
+      client is only doing a C(GET) or C(HEAD) on the URI to which it is being
+      redirected.
+    choices:
+    - all
+    - none
+    - safe
+    default: safe
+    type: str
+  headers:
+    description:
+    - Extra headers to set on the request.
+    - This should be a dictionary where the key is the header name and the
+      value is the value for that header.
+    type: dict
+  http_agent:
+    description:
+    - Header to identify as, generally appears in web server logs.
+    - This is set to the C(User-Agent) header on a HTTP request.
+    default: ansible-httpget
+    type: str
+    version_added: "2.9"
+  maximum_redirection:
+    description:
+    - Specify how many times the module will redirect a connection to an
+      alternative URI before the connection fails.
+    - If set to C(0) or I(follow_redirects) is set to C(none), or C(safe) when
+      not doing a C(GET) or C(HEAD) it prevents all redirection.
+    default: 50
+    type: int
+  timeout:
+    description:
+    - Specifies how long the request can be pending before it times out (in
+      seconds).
+    - Set to C(0) to specify an infinite timeout.
+    default: 30
+    type: int
+    version_added: "2.4"
+  validate_certs:
+    description:
+    - If C(no), SSL certificates will not be validated.
+    - This should only be used on personally controlled sites using self-signed
+      certificates.
+    default: yes
+    type: bool
+    version_added: "2.4"
+  client_cert:
+    description:
+    - The path to the client certificate (.pfx) that is used for X509
+      authentication. This path can either be the path to the C(pfx) on the
+      filesystem or the PowerShell certificate path
+      C(Cert:\CurrentUser\My\<thumbprint>).
+    - The WinRM connection must be authenticated with C(CredSSP) or C(become)
+      is used on the task if the certificate file is not password protected.
+    - Other authentication types can set I(client_cert_password) when the cert
+      is password protected.
+    type: str
+  client_cert_password:
+    description:
+    - The password for I(client_cert) if the cert is password protected.
+    type: str
+  force_basic_auth:
+    description:
+    - By default the authentication header is only sent when a webservice
+      responses to an initial request with a 401 status. Since some basic auth
+      services do not properly send a 401, logins will fail.
+    - This option forces the sending of the Basic authentication header upon
+      the original request.
+    default: no
+    type: bool
+    version_added: "2.5"
+  url_username:
+    description:
+    - The username to use for authentication.
+    type: str
+    aliases:
+    - user
+    - username
+  url_password:
+    description:
+    - The password for I(url_username).
+    type: str
+    aliases:
+    - password
+  use_default_credential:
+    description:
+    - Uses the current user's credentials when authenticating with a server
+      protected with C(NTLM), C(Kerberos), or C(Negotiate) authentication.
+    - Sites that use C(Basic) auth will still require explicit credentials
+      through the I(url_username) and I(url_password) options.
+    - The module will only have access to the user's credentials if using
+      C(become) with a password, you are connecting with SSH using a password,
+      or connecting with WinRM using C(CredSSP) or C(Kerberos with delegation).
+    - If not using C(become) or a different auth method to the ones stated
+      above, there will be no default credentials available and no
+      authentication will occur.
+    default: no
+    type: bool
+    version_added: "2.9"
+  use_proxy:
+    description:
+    - If C(no), it will not use the proxy defined in IE for the current user.
+    default: yes
+    type: bool
+  proxy_url:
+    description:
+    - An explicit proxy to use for the request.
+    - By default, the request will use the IE defined proxy unless I(use_proxy)
+      is set to C(no).
+    type: str
+  proxy_username:
+    description:
+    - The username to use for proxy authentication.
+    type: str
+  proxy_password:
+    description:
+    - The password for I(proxy_username).
+    type: str
+  proxy_use_default_credential:
+    description:
+    - Uses the current user's credentials when authenticating with a proxy host
+      protected with C(NTLM), C(Kerberos), or C(Negotiate) authentication.
+    - Proxies that use C(Basic) auth will still require explicit credentials
+      through the I(proxy_username) and I(proxy_password) options.
+    - The module will only have access to the user's credentials if using
+      C(become) with a password, you are connecting with SSH using a password,
+      or connecting with WinRM using C(CredSSP) or C(Kerberos with delegation).
+    - If not using C(become) or a different auth method to the ones stated
+      above, there will be no default credentials available and no proxy
+      authentication will occur.
+    default: no
+    type: bool
+    version_added: "2.9"
+seealso:
+- module: win_inet_proxy
+'''

test/integration/targets/win_get_url/tasks/main.yml

@@ -1,3 +1,5 @@
+# Due to the special environment setup required for proxies, you can manually test out the proxy options with
+# https://github.com/jborean93/ansible-powershell-url-proxy.
 ---
 - name: set fact out special testing dir
   set_fact:

test/integration/targets/win_get_url/tasks/tests_url.yml

@@ -180,7 +180,7 @@
     dest: '{{ testing_dir }}\timeout.txt'
     timeout: 3
   register: timeout_req
-  failed_when: timeout_req.msg != "Error requesting 'https://" + httpbin_host + "/delay/7'. The operation has timed out"
+  failed_when: 'timeout_req.msg != "Error downloading ''https://" + httpbin_host + "/delay/7'' to ''" + testing_dir + "\\timeout.txt'': The operation has timed out"'
 
 - name: send request with headers
   win_get_url:

test/integration/targets/win_uri/tasks/main.yml

@@ -443,3 +443,43 @@
     - not content_array is changed
     - content_array.content == '[{"abc":"def"}]'
     - content_array.json == [{"abc":"def"}]
+
+- name: send request with explicit http_agent
+  win_uri:
+    url: https://{{httpbin_host}}/get
+    http_agent: test-agent
+    return_content: yes
+  register: http_agent_option
+
+- name: assert send request with explicit http_agent
+  assert:
+    that:
+    - http_agent_option.json.headers['User-Agent'] == 'test-agent'
+
+- name: send request with explicit User-Agent header
+  win_uri:
+    url: https://{{httpbin_host}}/get
+    headers:
+      User-Agent: test-agent
+    return_content: yes
+  register: http_agent_header
+
+- name: assert send request with explicit User-Agent header
+  assert:
+    that:
+    - http_agent_header.json.headers['User-Agent'] == 'test-agent'
+
+- name: send request with explicit http_agent and header (http_agent wins)
+  win_uri:
+    url: https://{{httpbin_host}}/get
+    http_agent: test-agent-option
+    headers:
+      User-Agent: test-agent-header
+    return_content: yes
+  register: http_agent_combo
+
+- name: assert send request with explicit http_agent and header (http_agent wins)
+  assert:
+    that:
+    - http_agent_combo.json.headers['User-Agent'] == 'test-agent-option'
+    - http_agent_combo.warnings[0] == "The 'User-Agent' header and the 'http_agent' was set, using the 'http_agent' for web request"

test/sanity/pslint/ignore.txt

@@ -35,7 +35,6 @@ lib/ansible/modules/windows/win_eventlog.ps1 PSCustomUseLiteralPath
 lib/ansible/modules/windows/win_feature.ps1 PSCustomUseLiteralPath
 lib/ansible/modules/windows/win_file_version.ps1 PSCustomUseLiteralPath
 lib/ansible/modules/windows/win_firewall_rule.ps1 PSUseApprovedVerbs
-lib/ansible/modules/windows/win_get_url.ps1 PSUsePSCredentialType  # Credential param can take a base64 encoded string as well as a PSCredential
 lib/ansible/modules/windows/win_hotfix.ps1 PSCustomUseLiteralPath
 lib/ansible/modules/windows/win_hotfix.ps1 PSUseApprovedVerbs
 lib/ansible/modules/windows/win_iis_virtualdirectory.ps1 PSCustomUseLiteralPath