From 464ded80f563483327208313e6bc78ce2cff2c4f Mon Sep 17 00:00:00 2001 From: Kellin Date: Mon, 13 Nov 2017 11:13:25 -0500 Subject: [PATCH] Add link dest to synchronize module (#32746) * Clean up after two recent synchronize tests - add clean up after the last two tests in synchronize to make them match with the expectations in the previous tests Signed-off-by: Robert Marshall * Add link-dest functionality to synchronize module - add the link-dest option to the synchronize module code - add tests for the link-dest option - add documentation of the link_dest option to synchronize - modify changed flag so it can properly work around rsync upstream not flagging hardlinks as a change properly in formatted output Signed-off-by: Robert Marshall * Minor change to test --- lib/ansible/modules/files/synchronize.py | 36 ++++++++- .../targets/synchronize/tasks/main.yml | 79 +++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/lib/ansible/modules/files/synchronize.py b/lib/ansible/modules/files/synchronize.py index b7d7c07bbbe..35c755520b7 100644 --- a/lib/ansible/modules/files/synchronize.py +++ b/lib/ansible/modules/files/synchronize.py @@ -150,6 +150,12 @@ options: description: - Specify the private key to use for SSH-based rsync connections (e.g. C(~/.ssh/id_rsa)) version_added: "1.6" + link_dest: + description: + - add a destination to hard link against during the rsync. + default: + version_added: "2.5" + notes: - rsync must be installed on both the local and remote host. - For the C(synchronize) module, the "local host" is the host `the synchronize task originates on`, and the "destination host" is the host @@ -175,6 +181,8 @@ notes: rsync protocol in source or destination path. - The C(synchronize) module forces `--delay-updates` to avoid leaving a destination in a broken in-between state if the underlying rsync process encounters an error. Those synchronizing large numbers of files that are willing to trade safety for performance should call rsync directly. + - link_destination is subject to the same limitations as the underlaying rsync daemon. Hard links are only preserved if the relative subtrees + of the source and destination are the same. Attempts to hardlink into a directory that is a subdirectory of the source will be prevented. author: - Timothy Appnel (@tima) @@ -286,6 +294,13 @@ EXAMPLES = ''' rsync_opts: - "--no-motd" - "--exclude=.git" + +# Hardlink files if they didn't change +- name: Use hardlinks when synchronizing filesystems + synchronize: + src: /tmp/path_a/foo.txt + dest: /tmp/path_b/foo.txt + link_dest: /tmp/path_a/ ''' @@ -362,6 +377,7 @@ def main(): partial=dict(type='bool', default=False), verify_host=dict(type='bool', default=False), mode=dict(type='str', default='push', choices=['pull', 'push']), + link_dest=dict(type='list') ), supports_check_mode=True, ) @@ -398,6 +414,7 @@ def main(): rsync_opts = module.params['rsync_opts'] ssh_args = module.params['ssh_args'] verify_host = module.params['verify_host'] + link_dest = module.params['link_dest'] if '/' not in rsync: rsync = module.get_bin_path(rsync, required=True) @@ -475,6 +492,18 @@ def main(): if partial: cmd.append('--partial') + if link_dest: + cmd.append('-H') + # verbose required because rsync does not believe that adding a + # hardlink is actually a change + cmd.append('-vv') + for x in link_dest: + link_path = os.path.abspath(os.path.expanduser(x)) + destination_path = os.path.abspath(os.path.dirname(dest)) + if destination_path.find(link_path) == 0: + module.fail_json(msg='Hardlinking into a subdirectory of the source would cause recursion. %s and %s' % (destination_path, dest)) + cmd.append('--link-dest=%s' % link_path) + changed_marker = '<>' cmd.append('--out-format=' + changed_marker + '%i %n%L') @@ -491,7 +520,12 @@ def main(): if rc: return module.fail_json(msg=err, rc=rc, cmd=cmdstr) - changed = changed_marker in out + if link_dest: + # a leading period indicates no change + changed = (changed_marker + '.') not in out + else: + changed = changed_marker in out + out_clean = out.replace(changed_marker, '') out_lines = out_clean.split('\n') while '' in out_lines: diff --git a/test/integration/targets/synchronize/tasks/main.yml b/test/integration/targets/synchronize/tasks/main.yml index 64857823eff..5913762183e 100644 --- a/test/integration/targets/synchronize/tasks/main.yml +++ b/test/integration/targets/synchronize/tasks/main.yml @@ -173,6 +173,14 @@ - "sync_result.results[0].msg.endswith('+ foo.txt\n')" - "sync_result.results[1].msg.endswith('+ bar.txt\n')" +- name: Cleanup + file: + state: absent + path: "{{output_dir}}/{{item}}.result" + with_items: + - foo.txt + - bar.txt + - name: synchronize files using rsync_path (issue#7182) synchronize: src={{output_dir}}/foo.txt dest={{output_dir}}/foo.rsync_path rsync_path="sudo rsync" register: sync_result @@ -187,3 +195,74 @@ - "'msg' in sync_result" - "sync_result.msg.startswith('>f+')" - "sync_result.msg.endswith('+ foo.txt\n')" + +- name: Cleanup + file: + state: absent + path: "{{output_dir}}/{{item}}" + with_items: + - foo.rsync_path + +- name: add subdirectories for link-dest test + file: + path: "{{output_dir}}/{{item}}/" + state: directory + mode: 0755 + with_items: + - directory_a + - directory_b + +- name: copy foo.txt into the first directory + synchronize: + src: "{{output_dir}}/foo.txt" + dest: "{{output_dir}}/{{item}}/foo.txt" + with_items: + - directory_a + +- name: synchronize files using link_dest + synchronize: + src: "{{output_dir}}/directory_a/foo.txt" + dest: "{{output_dir}}/directory_b/foo.txt" + link_dest: + - "{{output_dir}}/directory_a" + register: sync_result + +- name: get stat information for directory_a + stat: + path: "{{ output_dir }}/directory_a/foo.txt" + register: stat_result_a + +- name: get stat information for directory_b + stat: + path: "{{ output_dir }}/directory_b/foo.txt" + register: stat_result_b + +- assert: + that: + - "'changed' in sync_result" + - "sync_result.changed == true" + - "stat_result_a.stat.inode == stat_result_b.stat.inode" + +- name: synchronize files using link_dest that would be recursive + synchronize: + src: "{{output_dir}}/foo.txt" + dest: "{{output_dir}}/foo.result" + link_dest: + - "{{output_dir}}" + register: sync_result + ignore_errors: yes + +- assert: + that: + - sync_result is not changed + - sync_result is failed + +- name: Cleanup + file: + state: absent + path: "{{output_dir}}/{{item}}" + with_items: + - "directory_b/foo.txt" + - "directory_a/foo.txt" + - "directory_a" + - "directory_b"