From 7c8e365dff0e5925a26fff6b3365b8fde867860b Mon Sep 17 00:00:00 2001 From: Jason Vanderhoof Date: Tue, 23 Jan 2018 09:04:57 -0700 Subject: [PATCH] Conjur Lookup Plugin (#34280) * Imported lookup plugin from Role * Plugin cleanup, including: * Use existing Python YAML parsing * Remove environment variables as connection options * Added initial debugging information * Reworked the lookup plugin using the Python Request library. As it's available through Ansible, it makes communication with Conjur much more straight forward. * Removed un-used libraries * Fixed linting issues * Standardized output on `format` and insure it works for 2.6, 2.7, and 3.x. * Use quote_plus from the six library for improved python 2/3 behavior. * Refactored identity & configuration to prefer user's file. This also includes a refactor to remove an un-needed dictionary merge method. * Removed `requests` in favor of `ansible.module_utils.urls`. * Refactored netrc loading to warn if host is not present. * Tests and a refactor to support easier testing. * Added reference to website * Fixed two linting errors * Fixed an extra line found by linting * Updated file write to use binary to insure config files are written correctly * Resolved linting issues * Refactored config & identity loading to take advantage of plugin options * Cleanup a bunch of small items caught by linting * Removed extra line caught by linting * Swapped in pytest and added some tests with mocked network responses * Pushing to see if this approach works better... * Refactored be open_url mocking based on feedback * Fixed a couple linting issues & refactored mocking into each method to attempt to resolve a failing test * Use a generic MagicMock for python 2.6 * Fixes doc typo require -> required * Use `type: path` in identity_file and config_file Also removes `expanduser` calls below (which will now be called automatically on paths.) * Defines maintainers for conjur_variable plugin * BOTMETA.yml: ** defines $team_cyberark_conjur as maintainers of Conjur Variable plugin ** adds myself and @jvanderhoof to that team * Adds URLs to relevant documentation for Conjur Variable lookup plugin * Clarifies "the server," "the machine" -> "controlling host" The machine identity used is that of the Ansible controlling host, not any server being provisioned or instructed. This documentation change aims to make that relationship clear. * Adds response code to exception message on authentication failure * Enhances exception messages to specify the controlling host These error messages are less likely to confuse a user as to which machine is associated with the files, identities, and configurations being described. * Adds ANSIBLE_METADATA for Conjur variable lookup plugin --- .github/BOTMETA.yml | 3 + lib/ansible/plugins/lookup/conjur_variable.py | 167 ++++++++++++++++++ .../plugins/lookup/test_conjur_variable.py | 110 ++++++++++++ 3 files changed, 280 insertions(+) create mode 100644 lib/ansible/plugins/lookup/conjur_variable.py create mode 100644 test/units/plugins/lookup/test_conjur_variable.py diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index c6e1ac7b30f..06445ddb9ed 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -1147,6 +1147,8 @@ files: lib/ansible/plugins/lookup/dig: maintainers: jpmens labels: community + lib/ansible/plugins/lookup/conjur_variable.py: + maintainers: $team_cyberark_conjur lib/ansible/plugins/netconf/: maintainers: $team_networking labels: networking @@ -1233,6 +1235,7 @@ macros: team_avi: ericsysmin grastogi23 khaltore team_azure: haroldwongms nitzmahone trstringer yuwzho xscript zikalino team_cumulus: isharacomix jrrivers privateip + team_cyberark_conjur: jvanderhoof ryanprior team_manageiq: gtanzillo abellotti zgalor yaacov cben team_netapp: hulquest lmprice broncofan gouthampacha team_netscaler: chiradeep giorgos-nikolopoulos diff --git a/lib/ansible/plugins/lookup/conjur_variable.py b/lib/ansible/plugins/lookup/conjur_variable.py new file mode 100644 index 00000000000..61830afd406 --- /dev/null +++ b/lib/ansible/plugins/lookup/conjur_variable.py @@ -0,0 +1,167 @@ +# (c) 2018, Jason Vanderhoof , Oren Ben Meir +# (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = """ + lookup: conjur_variable + version_added: "2.5" + short_description: Fetch credentials from CyberArk Conjur. + description: + - Retrieves credentials from Conjur using the controlling host's Conjur identity. Conjur info: U(https://www.conjur.org/). + requirements: + - The controlling host running Ansible has a Conjur identity. (More: U(https://developer.conjur.net/key_concepts/machine_identity.html)) + options: + _term: + description: Variable path + required: True + identity_file: + description: Path to the Conjur identity file. The identity file follows the netrc file format convention. + type: path + default: /etc/conjur.identity + required: False + ini: + - section: conjur, + key: identity_file_path + env: + - name: CONJUR_IDENTITY_FILE + config_file: + description: Path to the Conjur configuration file. The configuration file is a YAML file. + type: path + default: /etc/conjur.conf + required: False + ini: + - section: conjur, + key: config_file_path + env: + - name: CONJUR_CONFIG_FILE +""" + +EXAMPLES = """ + - debug + msg: {{ lookup('conjur_variable', '/path/to/secret') }} +""" + +RETURN = """ + _raw: + description: + - Value stored in Conjur. +""" + +import os.path +from ansible.errors import AnsibleError +from ansible.plugins.lookup import LookupBase +from base64 import b64encode +from netrc import netrc +from os import environ +from time import time +from ansible.module_utils.six.moves.urllib.parse import quote_plus +import yaml + +from ansible.module_utils.urls import open_url + + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +# Load configuration and return as dictionary if file is present on file system +def _load_conf_from_file(conf_path): + display.vvv('conf file: {0}'.format(conf_path)) + + if not os.path.exists(conf_path): + raise AnsibleError('Conjur configuration file `{0}` was not found on the controlling host' + .format(conf_path)) + + display.vvvv('Loading configuration from: {0}'.format(conf_path)) + with open(conf_path) as f: + config = yaml.safe_load(f.read()) + if 'account' not in config or 'appliance_url' not in config: + raise AnsibleError('{0} on the controlling host must contain an `account` and `appliance_url` entry' + .format(conf_path)) + return config + + +# Load identity and return as dictionary if file is present on file system +def _load_identity_from_file(identity_path, appliance_url): + display.vvvv('identity file: {0}'.format(identity_path)) + + if not os.path.exists(identity_path): + raise AnsibleError('Conjur identity file `{0}` was not found on the controlling host' + .format(identity_path)) + + display.vvvv('Loading identity from: {0} for {1}'.format(identity_path, appliance_url)) + + conjur_authn_url = '{0}/authn'.format(appliance_url) + identity = netrc(identity_path) + + if identity.authenticators(conjur_authn_url) is None: + raise AnsibleError('The netrc file on the controlling host does not contain an entry for: {0}' + .format(conjur_authn_url)) + + id, account, api_key = identity.authenticators(conjur_authn_url) + if not id or not api_key: + raise AnsibleError('{0} on the controlling host must contain a `login` and `password` entry for {1}' + .format(identity_path, appliance_url)) + + return {'id': id, 'api_key': api_key} + + +# Use credentials to retrieve temporary authorization token +def _fetch_conjur_token(conjur_url, account, username, api_key): + conjur_url = '{0}/authn/{1}/{2}/authenticate'.format(conjur_url, account, username) + display.vvvv('Authentication request to Conjur at: {0}, with user: {1}'.format(conjur_url, username)) + + response = open_url(conjur_url, data=api_key, method='POST') + code = response.getcode() + if code != 200: + raise AnsibleError('Failed to authenticate as \'{0}\' (got {1} response)' + .format(username, code)) + + return response.read() + + +# Retrieve Conjur variable using the temporary token +def _fetch_conjur_variable(conjur_variable, token, conjur_url, account): + token = b64encode(token) + headers = {'Authorization': 'Token token="{0}"'.format(token)} + display.vvvv('Header: {0}'.format(headers)) + + url = '{0}/secrets/{1}/variable/{2}'.format(conjur_url, account, quote_plus(conjur_variable)) + display.vvvv('Conjur Variable URL: {0}'.format(url)) + + response = open_url(url, headers=headers, method='GET') + + if response.getcode() == 200: + display.vvvv('Conjur variable {0} was successfully retrieved'.format(conjur_variable)) + return [response.read()] + if response.getcode() == 401: + raise AnsibleError('Conjur request has invalid authorization credentials') + if response.getcode() == 403: + raise AnsibleError('The controlling host\'s Conjur identity does not have authorization to retrieve {0}' + .format(conjur_variable)) + if response.getcode() == 404: + raise AnsibleError('The variable {0} does not exist'.format(conjur_variable)) + + return {} + + +class LookupModule(LookupBase): + + def run(self, terms, variables=None, **kwargs): + conf_file = self.get_option('config_file') + conf = _load_conf_from_file(conf_file) + + identity_file = self.get_option('identity_file') + identity = _load_identity_from_file(identity_file, conf['appliance_url']) + + token = _fetch_conjur_token(conf['appliance_url'], conf['account'], identity['id'], identity['api_key']) + return _fetch_conjur_variable(terms[0], token, conf['appliance_url'], conf['account']) diff --git a/test/units/plugins/lookup/test_conjur_variable.py b/test/units/plugins/lookup/test_conjur_variable.py new file mode 100644 index 00000000000..a5b5401704f --- /dev/null +++ b/test/units/plugins/lookup/test_conjur_variable.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# (c) 2018, Jason Vanderhoof +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +import pytest +from ansible.compat.tests.mock import MagicMock +from ansible.errors import AnsibleError +from ansible.module_utils.six.moves import http_client +from ansible.plugins.lookup import conjur_variable +import tempfile + + +class TestLookupModule: + def test_valid_netrc_file(self): + with tempfile.NamedTemporaryFile() as temp_netrc: + temp_netrc.write(b"machine http://localhost/authn\n") + temp_netrc.write(b" login admin\n") + temp_netrc.write(b" password my-pass\n") + temp_netrc.seek(0) + + results = conjur_variable._load_identity_from_file(temp_netrc.name, 'http://localhost') + + assert results['id'] == 'admin' + assert results['api_key'] == 'my-pass' + + def test_netrc_without_host_file(self): + with tempfile.NamedTemporaryFile() as temp_netrc: + temp_netrc.write(b"machine http://localhost/authn\n") + temp_netrc.write(b" login admin\n") + temp_netrc.write(b" password my-pass\n") + temp_netrc.seek(0) + + with pytest.raises(AnsibleError): + conjur_variable._load_identity_from_file(temp_netrc.name, 'http://foo') + + def test_valid_configuration(self): + with tempfile.NamedTemporaryFile() as configuration_file: + configuration_file.write(b"---\n") + configuration_file.write(b"account: demo-policy\n") + configuration_file.write(b"plugins: []\n") + configuration_file.write(b"appliance_url: http://localhost:8080\n") + configuration_file.seek(0) + + results = conjur_variable._load_conf_from_file(configuration_file.name) + assert results['account'] == 'demo-policy' + assert results['appliance_url'] == 'http://localhost:8080' + + def test_valid_token_retrieval(self, mocker): + mock_response = MagicMock(spec_set=http_client.HTTPResponse) + try: + mock_response.getcode.return_value = 200 + except: + # HTTPResponse is a Python 3 only feature. This uses a generic mock for python 2.6 + mock_response = MagicMock() + mock_response.getcode.return_value = 200 + + mock_response.read.return_value = 'foo-bar-token' + mocker.patch.object(conjur_variable, 'open_url', return_value=mock_response) + + response = conjur_variable._fetch_conjur_token('http://conjur', 'account', 'username', 'api_key') + assert response == 'foo-bar-token' + + def test_valid_fetch_conjur_variable(self, mocker): + mock_response = MagicMock(spec_set=http_client.HTTPResponse) + try: + mock_response.getcode.return_value = 200 + except: + # HTTPResponse is a Python 3 only feature. This uses a generic mock for python 2.6 + mock_response = MagicMock() + mock_response.getcode.return_value = 200 + + mock_response.read.return_value = 'foo-bar' + mocker.patch.object(conjur_variable, 'open_url', return_value=mock_response) + + response = conjur_variable._fetch_conjur_token('super-secret', 'token', 'http://conjur', 'account') + assert response == 'foo-bar' + + def test_invalid_fetch_conjur_variable(self, mocker): + for code in [401, 403, 404]: + mock_response = MagicMock(spec_set=http_client.HTTPResponse) + try: + mock_response.getcode.return_value = code + except: + # HTTPResponse is a Python 3 only feature. This uses a generic mock for python 2.6 + mock_response = MagicMock() + mock_response.getcode.return_value = code + + mocker.patch.object(conjur_variable, 'open_url', return_value=mock_response) + + with pytest.raises(AnsibleError): + response = conjur_variable._fetch_conjur_token('super-secret', 'token', 'http://conjur', 'account')