Implement part of #27617 [expend checksum format to <algorithm>:(<checksum>|<url>)] (#43751)

* expend checksum format to <algorithm>:(<checksum>|<url>)

* continue to code at office

* ALPHA - expend checksum format to <algorithm>:(<checksum>|<url>)

* clean up tmpfile and comment

* try to add test code for 27617

* try to add test code for 27617

* try to add test code for 27617

* try to fix [Could not find or access 'testserver.py']

* fix test code [Could not find or access 'testserver.py']

* fix test code [add files dir]

* fix test code [files dir not exists]

* as [connection was closed before a valid response was received]

* [connection was closed before a valid response was received]

* [connection was closed before a valid response was received]

* add test item [sha1 and sha256]

* since [connection was closed before a valid response was received]

* fix [connection was closed before a valid response was received]

* fix test code typo

* add docs for #27617

* PR #43751 is minor change

* fix pep8 issue.

* fix test code style.

* fix unexpected quote

changelogs/fragments/get_url.yaml

@@ -0,0 +1,5 @@
+minor_changes:
+- get_url - implement [expend checksum format to <algorithm>:(<checksum>|<url>)] (https://github.com/ansible/ansible/issues/27617)
+
+bugfixes:
+- get_url - fix the bug that get_url does not change mode when checksum matches (https://github.com/ansible/ansible/issues/29614)

changelogs/fragments/get_url_bug_fix.yaml

@@ -1,2 +0,0 @@
-bugfixes:
-- get_url - fix the bug that get_url does not change mode when checksum matches (https://github.com/ansible/ansible/issues/29614)

lib/ansible/modules/net_tools/basics/get_url.py

@@ -85,7 +85,8 @@ options:
       - 'If a checksum is passed to this parameter, the digest of the
         destination file will be calculated after it is downloaded to ensure
         its integrity and verify that the transfer completed successfully.
-        Format: <algorithm>:<checksum>, e.g. checksum="sha256:D98291AC[...]B6DC7B97"'
+        Format: <algorithm>:<checksum|url>, e.g. checksum="sha256:D98291AC[...]B6DC7B97",
+        checksum="sha256:http://example.com/path/sha256sum.txt"'
       - If you worry about portability, only the sha1 algorithm is available
         on all platforms and python versions.
       - The third party hashlib library can be installed for access to additional algorithms.
@@ -192,6 +193,12 @@ EXAMPLES = r'''
     dest: /etc/foo.conf
     checksum: md5:66dffb5228a211e61d6d7ef4a86f5758
 
+- name: Download file with checksum url (sha256)
+  get_url:
+    url: http://example.com/path/file.conf
+    dest: /etc/foo.conf
+    checksum: 'sha256:http://example.com/path/sha256sum.txt'
+
 - name: Download file from a file path
   get_url:
     url: file:///tmp/afile.txt
@@ -433,7 +440,16 @@ def main():
     # checksum specified, parse for algorithm and checksum
     if checksum:
         try:
-            algorithm, checksum = checksum.rsplit(':', 1)
+            algorithm, checksum = checksum.split(':', 1)
+            if checksum.startswith('http://') or checksum.startswith('https://') or checksum.startswith('ftp://'):
+                checksum_url = checksum
+                # download checksum file to checksum_tmpsrc
+                checksum_tmpsrc, checksum_info = url_get(module, checksum_url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest)
+                lines = [line.rstrip('\n') for line in open(checksum_tmpsrc)]
+                os.remove(checksum_tmpsrc)
+                lines = dict(s.split(None, 1) for s in lines)
+                filename = url_filename(url)
+                [checksum] = (k for (k, v) in lines.items() if v == filename)
             # Remove any non-alphanumeric characters, including the infamous
             # Unicode zero-width space
             checksum = re.sub(r'\W+', '', checksum).lower()

test/integration/targets/get_url/files/testserver.py

@@ -0,0 +1,20 @@
+import sys
+
+if __name__ == '__main__':
+    if sys.version_info[0] >= 3:
+        import http.server
+        import socketserver
+        PORT = int(sys.argv[1])
+
+        class Handler(http.server.SimpleHTTPRequestHandler):
+            pass
+
+        Handler.extensions_map['.json'] = 'application/json'
+        httpd = socketserver.TCPServer(("", PORT), Handler)
+        httpd.serve_forever()
+    else:
+        import mimetypes
+        mimetypes.init()
+        mimetypes.add_type('application/json', '.json')
+        import SimpleHTTPServer
+        SimpleHTTPServer.test()

test/integration/targets/get_url/tasks/main.yml

@@ -257,6 +257,88 @@
       - result is changed
       - "stat_result.stat.mode == '0775'"
 
+# https://github.com/ansible/ansible/issues/27617
+
+- name: set role facts
+  set_fact:
+    http_port: 27617
+    files_dir: '{{ output_dir }}/files'
+
+- name: create files_dir
+  file:
+    dest: "{{ files_dir }}"
+    state: directory
+
+- name: create src file
+  copy:
+    dest: '{{ files_dir }}/27617.txt'
+    content: "ptux"
+
+- name: create sha1 checksum file of src
+  copy:
+    dest: '{{ files_dir }}/sha1sum.txt'
+    content: "a97e6837f60cec6da4491bab387296bbcd72bdba  27617.txt"
+
+- name: add sha1 checksum not going to be downloaded
+  lineinfile:
+    dest: "{{ files_dir }}/sha1sum.txt"
+    line: "{{ item }}"
+  loop:
+    - '3911340502960ca33aece01129234460bfeb2791  not_target1.txt'
+    - '1b4b6adf30992cedb0f6edefd6478ff0a593b2e4  not_target2.txt'
+
+- name: create sha256 checksum file of src
+  copy:
+    dest: '{{ files_dir }}/sha256sum.txt'
+    content: "b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  27617.txt"
+
+- name: add sha256 checksum not going to be downloaded
+  lineinfile:
+    dest: "{{ files_dir }}/sha256sum.txt"
+    line: "{{ item }}"
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  not_target2.txt'
+
+- copy:
+    src: "testserver.py"
+    dest: "{{ output_dir }}/testserver.py"
+
+- name: start SimpleHTTPServer for issues 27617
+  shell: cd {{ files_dir }} && {{ ansible_python.executable }} {{ output_dir}}/testserver.py {{ http_port }}
+  async: 90
+  poll: 0
+
+- name: download src with sha1 checksum url
+  get_url:
+    url: 'http://localhost:{{ http_port }}/27617.txt'
+    dest: '{{ output_dir }}'
+    checksum: 'sha1:http://localhost:{{ http_port }}/sha1sum.txt'
+  register: result_sha1
+
+- stat:
+    path: "{{ output_dir }}/27617.txt"
+  register: stat_result_sha1
+
+- name: download src with sha256 checksum url
+  get_url:
+    url: 'http://localhost:{{ http_port }}/27617.txt'
+    dest: '{{ output_dir }}/27617sha256.txt'
+    checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum.txt'
+  register: result_sha256
+
+- stat:
+    path: "{{ output_dir }}/27617.txt"
+  register: stat_result_sha256
+
+- name: Assert that the file was downloaded
+  assert:
+    that:
+      - result_sha1 is changed
+      - result_sha256 is changed
+      - "stat_result_sha1.stat.exists == true"
+      - "stat_result_sha256.stat.exists == true"
+
 #https://github.com/ansible/ansible/issues/16191
 - name: Test url split with no filename
   get_url: