diff --git a/changelogs/fragments/64288-fix-hashi-vault-kv-v2.yaml b/changelogs/fragments/64288-fix-hashi-vault-kv-v2.yaml new file mode 100644 index 00000000000..5529f265ce0 --- /dev/null +++ b/changelogs/fragments/64288-fix-hashi-vault-kv-v2.yaml @@ -0,0 +1,2 @@ +bugfixes: +- "hashi_vault - Fix KV v2 lookup to always return latest version" diff --git a/docs/docsite/rst/porting_guides/porting_guide_2.10.rst b/docs/docsite/rst/porting_guides/porting_guide_2.10.rst index e62dc037c4a..ffc3f706e68 100644 --- a/docs/docsite/rst/porting_guides/porting_guide_2.10.rst +++ b/docs/docsite/rst/porting_guides/porting_guide_2.10.rst @@ -102,8 +102,11 @@ Noteworthy module changes Plugins ======= -No notable changes +Noteworthy plugin changes +------------------------- + +* The ``hashi_vault`` lookup plugin now returns the latest version when using the KV v2 secrets engine. Previously, it returned all versions of the secret which required additional steps to extract and filter the desired version. Porting custom scripts ====================== diff --git a/lib/ansible/plugins/lookup/hashi_vault.py b/lib/ansible/plugins/lookup/hashi_vault.py index 473872d4ada..0f54526dfe3 100644 --- a/lib/ansible/plugins/lookup/hashi_vault.py +++ b/lib/ansible/plugins/lookup/hashi_vault.py @@ -16,6 +16,7 @@ DOCUMENTATION = """ - retrieve secrets from HashiCorp's vault notes: - Due to a current limitation in the HVAC library there won't necessarily be an error if a bad endpoint is specified. + - As of Ansible 2.10, only the latest secret is returned when specifying a KV v2 path. options: secret: description: query you are making. @@ -98,8 +99,9 @@ EXAMPLES = """ debug: msg: "{{ lookup('hashi_vault', 'secret=secret/hello token=c975b780-d1be-8016-866b-01d0f9b688a5 url=http://myvault:8200 namespace=teama/admins')}}" -# to work with kv v2 (vault api - for kv v2 - GET method requires that PATH should be "secret/data/:path") -- name: Return all kv v2 secrets from a path +# When using KV v2 the PATH should include "data" between the secret engine mount and path (e.g. "secret/data/:path") +# see: https://www.vaultproject.io/api/secret/kv/kv-v2.html#read-secret-version +- name: Return latest KV v2 secret from path debug: msg: "{{ lookup('hashi_vault', 'secret=secret/data/hello token=my_vault_token url=http://myvault_url:8200') }}" @@ -197,6 +199,18 @@ class HashiVault: def get(self): data = self.client.read(self.secret) + # Check response for KV v2 fields and flatten nested secret data. + # + # https://vaultproject.io/api/secret/kv/kv-v2.html#sample-response-1 + try: + # sentinel field checks + check_dd = data['data']['data'] + check_md = data['data']['metadata'] + # unwrap nested data + data = data['data'] + except KeyError: + pass + if data is None: raise AnsibleError("The secret %s doesn't seem to exist for hashi_vault lookup" % self.secret) diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml index 4d19195a395..f1f6dd981d4 100644 --- a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml +++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml @@ -1,3 +1,4 @@ --- -vault_base_path: 'secret/data/testproject' -vault_base_path_kv: 'secret/testproject' # required by KV 2 engine +vault_gen_path: 'gen/testproject' +vault_kv1_path: 'kv1/testproject' +vault_kv2_path: 'kv2/data/testproject' diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml index a97c427cff0..44eb5ed18d4 100644 --- a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml +++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml @@ -4,17 +4,17 @@ block: - name: 'Fetch secrets using "hashi_vault" lookup' set_fact: - secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret1 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" - secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" + secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret1 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" + secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" - name: 'Check secret values' fail: msg: 'unexpected secret values' - when: secret1['data']['value'] != 'foo1' or secret2['data']['value'] != 'foo2' + when: secret1['value'] != 'foo1' or secret2['value'] != 'foo2' - name: 'Failure expected when erroneous credentials are used' vars: - secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=approle secret_id=toto role_id=' ~ role_id) }}" + secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=approle secret_id=toto role_id=' ~ role_id) }}" debug: msg: 'Failure is expected ({{ secret_wrong_cred }})' register: test_wrong_cred @@ -22,7 +22,7 @@ - name: 'Failure expected when unauthorized secret is read' vars: - secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret3 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" + secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret3 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" debug: msg: 'Failure is expected ({{ secret_unauthorized }})' register: test_unauthorized @@ -30,7 +30,7 @@ - name: 'Failure expected when inexistent secret is read' vars: - secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret4 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" + secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret4 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}" debug: msg: 'Failure is expected ({{ secret_inexistent }})' register: test_inexistent diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml index 9bde696c5af..42fd0907f3d 100644 --- a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml +++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml @@ -74,22 +74,57 @@ - name: 'Start vault server (dev mode enabled)' shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl /dev/null 2>&1 &' + - name: 'Create generic secrets engine' + command: '{{ vault_cmd }} secrets enable -path=gen generic' + + - name: 'Create KV v1 secrets engine' + command: '{{ vault_cmd }} secrets enable -path=kv1 -version=1 kv' + + - name: 'Create KV v2 secrets engine' + command: '{{ vault_cmd }} secrets enable -path=kv2 -version=2 kv' + - name: 'Create a test policy' shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write test-policy -" vars: policy: | - path "{{ vault_base_path }}/secret1" { + path "{{ vault_gen_path }}/secret1" { capabilities = ["read"] } - path "{{ vault_base_path }}/secret2" { + path "{{ vault_gen_path }}/secret2" { capabilities = ["read", "update"] } - path "{{ vault_base_path }}/secret3" { + path "{{ vault_gen_path }}/secret3" { + capabilities = ["deny"] + } + path "{{ vault_kv1_path }}/secret1" { + capabilities = ["read"] + } + path "{{ vault_kv1_path }}/secret2" { + capabilities = ["read", "update"] + } + path "{{ vault_kv1_path }}/secret3" { + capabilities = ["deny"] + } + path "{{ vault_kv2_path }}/secret1" { + capabilities = ["read"] + } + path "{{ vault_kv2_path }}/secret2" { + capabilities = ["read", "update"] + } + path "{{ vault_kv2_path }}/secret3" { capabilities = ["deny"] } - - name: 'Create secrets' - command: '{{ vault_cmd }} kv put {{ vault_base_path_kv }}/secret{{ item }} value=foo{{ item }}' + - name: 'Create generic secrets' + command: '{{ vault_cmd }} write {{ vault_gen_path }}/secret{{ item }} value=foo{{ item }}' + loop: [1, 2, 3] + + - name: 'Create KV v1 secrets' + command: '{{ vault_cmd }} kv put {{ vault_kv1_path }}/secret{{ item }} value=foo{{ item }}' + loop: [1, 2, 3] + + - name: 'Create KV v2 secrets' + command: '{{ vault_cmd }} kv put {{ vault_kv2_path | regex_replace("/data") }}/secret{{ item }} value=foo{{ item }}' loop: [1, 2, 3] - name: setup approle auth diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml index 927881da898..20c1af791ee 100644 --- a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml +++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml @@ -3,18 +3,31 @@ block: - name: 'Fetch secrets using "hashi_vault" lookup' set_fact: - secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret1 auth_method=token token=' ~ user_token) }}" - secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 token=' ~ user_token) }}" - secret3: "{{ lookup('hashi_vault', conn_params ~ ' secret=' ~ vault_base_path ~ '/secret2 token=' ~ user_token) }}" + gen_secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_gen_path ~ '/secret1 auth_method=token token=' ~ user_token) }}" + gen_secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_gen_path ~ '/secret2 token=' ~ user_token) }}" + kv1_secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv1_path ~ '/secret1 auth_method=token token=' ~ user_token) }}" + kv1_secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv1_path ~ '/secret2 token=' ~ user_token) }}" + kv2_secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret1 auth_method=token token=' ~ user_token) }}" + kv2_secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 token=' ~ user_token) }}" - - name: 'Check secret values' + - name: 'Check secret generic values' fail: msg: 'unexpected secret values' - when: secret1['data']['value'] != 'foo1' or secret2['data']['value'] != 'foo2' or secret3['data']['value'] != 'foo2' + when: gen_secret1['value'] != 'foo1' or gen_secret2['value'] != 'foo2' + + - name: 'Check secret kv1 values' + fail: + msg: 'unexpected secret values' + when: kv1_secret1['value'] != 'foo1' or kv1_secret2['value'] != 'foo2' + + - name: 'Check secret kv2 values' + fail: + msg: 'unexpected secret values' + when: kv2_secret1['value'] != 'foo1' or kv2_secret2['value'] != 'foo2' - name: 'Failure expected when erroneous credentials are used' vars: - secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=token token=wrong_token') }}" + secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret2 auth_method=token token=wrong_token') }}" debug: msg: 'Failure is expected ({{ secret_wrong_cred }})' register: test_wrong_cred @@ -22,7 +35,7 @@ - name: 'Failure expected when unauthorized secret is read' vars: - secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret3 token=' ~ user_token) }}" + secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret3 token=' ~ user_token) }}" debug: msg: 'Failure is expected ({{ secret_unauthorized }})' register: test_unauthorized @@ -30,7 +43,7 @@ - name: 'Failure expected when inexistent secret is read' vars: - secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret4 token=' ~ user_token) }}" + secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_kv2_path ~ '/secret4 token=' ~ user_token) }}" debug: msg: 'Failure is expected ({{ secret_inexistent }})' register: test_inexistent