Support using vault password files that are themselves vault encrypted (#27668)

Extract vault related bits of DataLoader._get_file_contents to DataLoader._decrypt_if_vault_data

When loading vault password files, detect if they are vault encrypted, and if so, try to decrypt with any already known vault secrets.

This implements the 'Allow vault password files to be vault encrypted'  (#31002) feature card from
the 2.5.0 project at https://github.com/ansible/ansible/projects/9

Fixes #31002

lib/ansible/parsing/dataloader.py

@@ -18,7 +18,7 @@ from yaml import YAMLError
 from ansible.errors import AnsibleFileNotFound, AnsibleParserError
 from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
 from ansible.module_utils.basic import is_executable
-from ansible.module_utils.six import binary_type, string_types, text_type
+from ansible.module_utils.six import binary_type, text_type
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.parsing.quoting import unquote
 from ansible.parsing.vault import VaultLib, b_HEADER, is_encrypted, is_encrypted_file, parse_vaulttext_envelope
@@ -174,6 +174,19 @@ class DataLoader:
             except AttributeError:
                 pass  # older versions of yaml don't have dispose function, ignore
 
+    def _decrypt_if_vault_data(self, b_vault_data, b_file_name=None):
+        '''Decrypt b_vault_data if encrypted and return b_data and the show_content flag'''
+
+        if not is_encrypted(b_vault_data):
+            show_content = True
+            return b_vault_data, show_content
+
+        b_ciphertext, b_version, cipher_name, vault_id = parse_vaulttext_envelope(b_vault_data)
+        b_data = self._vault.decrypt(b_vault_data, filename=b_file_name)
+
+        show_content = False
+        return b_data, show_content
+
     def _get_file_contents(self, file_name):
         '''
         Reads the file contents from the given file name
@@ -196,17 +209,10 @@ class DataLoader:
         if not self.path_exists(b_file_name) or not self.is_file(b_file_name):
             raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name)
 
-        show_content = True
         try:
             with open(b_file_name, 'rb') as f:
                 data = f.read()
-                if is_encrypted(data):
-                    # FIXME: plugin vault selector
-                    b_ciphertext, b_version, cipher_name, vault_id = parse_vaulttext_envelope(data)
-                    data = self._vault.decrypt(data, filename=b_file_name)
-                    show_content = False
-
-            return (data, show_content)
+                return self._decrypt_if_vault_data(data, b_file_name)
 
         except (IOError, OSError) as e:
             raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)), orig_exc=e)

lib/ansible/parsing/vault/__init__.py

@@ -430,6 +430,10 @@ class FileVaultSecret(VaultSecret):
         except (OSError, IOError) as e:
             raise AnsibleError("Could not read vault password file %s: %s" % (filename, e))
 
+        b_vault_data, dummy = self.loader._decrypt_if_vault_data(vault_pass, filename)
+
+        vault_pass = b_vault_data.strip(b'\r\n')
+
         verify_secret_is_not_empty(vault_pass,
                                    msg='Invalid vault password was provided from file (%s)' % filename)
 

test/integration/targets/vault/encrypted-vault-password

@@ -0,0 +1,6 @@
+$ANSIBLE_VAULT;1.1;AES256
+34353166613539646338666531633061646161663836373965663032313466613135313130383133
+3634383331386336333436323832356264343033323166370a323737396234376132353731643863
+62386335616635363062613562666561643931626332623464306666636131356134386531363533
+3831323230353333620a616633376363373830346332663733316634663937336663633631326361
+62343638656532393932643530633133326233316134383036316333373962626164

test/integration/targets/vault/runme.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eux
+set -euvx
 
 MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
 trap 'rm -rf "${MYTMPDIR}"' EXIT
@@ -12,6 +12,12 @@ echo "This is a test file" > "${TEST_FILE}"
 TEST_FILE_1_2="${MYTMPDIR}/test_file_1_2"
 echo "This is a test file for format 1.2" > "${TEST_FILE_1_2}"
 
+TEST_FILE_ENC_PASSWORD="${MYTMPDIR}/test_file_enc_password"
+echo "This is a test file for encrypted with a vault password that is itself vault encrypted" > "${TEST_FILE_ENC_PASSWORD}"
+
+TEST_FILE_ENC_PASSWORD_DEFAULT="${MYTMPDIR}/test_file_enc_password_default"
+echo "This is a test file for encrypted with a vault password that is itself vault encrypted using --encrypted-vault-id default" > "${TEST_FILE_ENC_PASSWORD_DEFAULT}"
+
 TEST_FILE_OUTPUT="${MYTMPDIR}/test_file_output"
 
 TEST_FILE_EDIT="${MYTMPDIR}/test_file_edit"
@@ -20,6 +26,20 @@ echo "This is a test file for edit" > "${TEST_FILE_EDIT}"
 TEST_FILE_EDIT2="${MYTMPDIR}/test_file_edit2"
 echo "This is a test file for edit2" > "${TEST_FILE_EDIT2}"
 
+# view the vault encrypted password file
+ansible-vault view "$@" --vault-id vault-password encrypted-vault-password
+
+# encrypt with a password from a vault encrypted password file and multiple vault-ids
+# should fail because we dont know which vault id to use to encrypt with
+ansible-vault encrypt "$@" --vault-id vault-password --vault-id encrypted-vault-password "${TEST_FILE_ENC_PASSWORD}" && :
+WRONG_RC=$?
+echo "rc was $WRONG_RC (5 is expected)"
+[ $WRONG_RC -eq 5 ]
+
+# try to view the file encrypted with the vault-password we didnt specify
+# to verify we didnt choose the wrong vault-id
+ansible-vault view "$@" --vault-id vault-password encrypted-vault-password
+
 FORMAT_1_1_HEADER="\$ANSIBLE_VAULT;1.1;AES256"
 FORMAT_1_2_HEADER="\$ANSIBLE_VAULT;1.2;AES256"
 
@@ -30,9 +50,6 @@ ansible-vault view "$@" --vault-id vault-password@test-vault-client.py format_1_
 
 # view, using password client script, unknown vault/keyname
 ansible-vault view "$@" --vault-id some_unknown_vault_id@test-vault-client.py format_1_1_AES256.yml && :
-WRONG_RC=$?
-echo "rc was $WRONG_RC (1 is expected)"
-[ $WRONG_RC -eq 1 ]
 
 # Use linux setsid to test without a tty. No setsid if osx/bsd though...
 if [ -x "$(command -v setsid)" ]; then
@@ -319,6 +336,37 @@ head -1 "${TEST_FILE_EDIT2}" | grep "${FORMAT_1_2_HEADER};vault_password"
 EDITOR=./faux-editor.py ansible-vault edit "$@" --vault-password-file vault-password "${TEST_FILE_EDIT2}"
 head -1 "${TEST_FILE_EDIT2}" | grep "${FORMAT_1_2_HEADER};vault_password"
 
+# encrypt with a password from a vault encrypted password file and multiple vault-ids
+# should fail because we dont know which vault id to use to encrypt with
+ansible-vault encrypt "$@" --vault-id vault-password --vault-id encrypted-vault-password "${TEST_FILE_ENC_PASSWORD}" && :
+WRONG_RC=$?
+echo "rc was $WRONG_RC (5 is expected)"
+[ $WRONG_RC -eq 5 ]
+
+
+# encrypt with a password from a vault encrypted password file and multiple vault-ids
+# but this time specify with --encrypt-vault-id, but specifying vault-id names (instead of default)
+# ansible-vault encrypt "$@" --vault-id from_vault_password@vault-password --vault-id from_encrypted_vault_password@encrypted-vault-password --encrypt-vault-id from_encrypted_vault_password "${TEST_FILE_ENC_PASSWORD}"
+
+# try to view the file encrypted with the vault-password we didnt specify
+# to verify we didnt choose the wrong vault-id
+# ansible-vault view "$@" --vault-id vault-password "${TEST_FILE_ENC_PASSWORD}" && :
+# WRONG_RC=$?
+# echo "rc was $WRONG_RC (1 is expected)"
+# [ $WRONG_RC -eq 1 ]
+
+ansible-vault encrypt "$@" --vault-id vault-password "${TEST_FILE_ENC_PASSWORD}"
+
+# view the file encrypted with a password from a vault encrypted password file
+ansible-vault view "$@" --vault-id vault-password --vault-id encrypted-vault-password "${TEST_FILE_ENC_PASSWORD}"
+
+# try to view the file encrypted with a password from a vault encrypted password file but without the password to the password file.
+# This should fail with an
+ansible-vault view "$@" --vault-id encrypted-vault-password "${TEST_FILE_ENC_PASSWORD}" && :
+WRONG_RC=$?
+echo "rc was $WRONG_RC (1 is expected)"
+[ $WRONG_RC -eq 1 ]
+
 
 # test playbooks using vaulted files
 ansible-playbook test_vault.yml          -i ../../inventory -v "$@" --vault-password-file vault-password --list-tasks

test/units/parsing/vault/test_vault.py

@@ -148,6 +148,11 @@ class TestPromptVaultSecret(unittest.TestCase):
 
 
 class TestFileVaultSecret(unittest.TestCase):
+    def setUp(self):
+        self.vault_password = "test-vault-password"
+        text_secret = TextVaultSecret(self.vault_password)
+        self.vault_secrets = [('foo', text_secret)]
+
     def test(self):
         secret = vault.FileVaultSecret()
         self.assertIsNone(secret._bytes)
@@ -201,6 +206,36 @@ class TestFileVaultSecret(unittest.TestCase):
 
         os.unlink(tmp_file.name)
 
+    def test_file_encrypted(self):
+        vault_password = "test-vault-password"
+        text_secret = TextVaultSecret(vault_password)
+        vault_secrets = [('foo', text_secret)]
+
+        password = 'some password'
+        # 'some password' encrypted with 'test-ansible-password'
+
+        password_file_content = '''$ANSIBLE_VAULT;1.1;AES256
+61393863643638653437313566313632306462383837303132346434616433313438353634613762
+3334363431623364386164616163326537366333353663650a663634306232363432626162353665
+39623061353266373631636331643761306665343731376633623439313138396330346237653930
+6432643864346136640a653364386634666461306231353765636662316335613235383565306437
+3737
+'''
+
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        tmp_file.write(to_bytes(password_file_content))
+        tmp_file.close()
+
+        fake_loader = DictDataLoader({tmp_file.name: 'sdfadf'})
+        fake_loader._vault.secrets = vault_secrets
+
+        secret = vault.FileVaultSecret(loader=fake_loader, filename=tmp_file.name)
+        secret.load()
+
+        os.unlink(tmp_file.name)
+
+        self.assertEqual(secret.bytes, to_bytes(password))
+
     def test_file_not_a_directory(self):
         filename = '/dev/null/foobar'
         fake_loader = DictDataLoader({filename: 'sdfadf'})